diff options
Diffstat (limited to 'krebs/3modules')
31 files changed, 437 insertions, 380 deletions
diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index 92400139c..d58661a28 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -1,6 +1,6 @@ { config, lib, pkgs, ... }: -with lib; +with config.krebs.lib; let ReaktorConfig = pkgs.writeText "config.py" '' @@ -16,7 +16,7 @@ let out = { options.krebs.Reaktor = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; }; api = { diff --git a/krebs/3modules/apt-cacher-ng.nix b/krebs/3modules/apt-cacher-ng.nix index 371d39b6f..46b405842 100644 --- a/krebs/3modules/apt-cacher-ng.nix +++ b/krebs/3modules/apt-cacher-ng.nix @@ -1,6 +1,6 @@ { config, pkgs, lib, ... }: -with lib; +with config.krebs.lib; let acng-config = pkgs.writeTextFile { name = "acng-configuration"; @@ -151,5 +151,5 @@ let in { options.krebs.apt-cacher-ng = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; } diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix index a1f335905..0aa86dec9 100644 --- a/krebs/3modules/backup.nix +++ b/krebs/3modules/backup.nix @@ -1,9 +1,9 @@ { config, lib, pkgs, ... }: -with lib; +with config.krebs.lib; let out = { options.krebs.backup = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; }; cfg = config.krebs.backup; @@ -12,14 +12,17 @@ let enable = mkEnableOption "krebs.backup" // { default = true; }; plans = mkOption { default = {}; - type = types.attrsOf (types.submodule ({ - # TODO enable = mkEnableOption "TODO" // { default = true; }; + type = types.attrsOf (types.submodule ({ config, ... }: { options = { + enable = mkEnableOption "krebs.backup.${config.name}" // { + default = true; + }; method = mkOption { type = types.enum ["pull" "push"]; }; name = mkOption { type = types.str; + default = config._module.args.name; }; src = mkOption { type = types.krebs.file-location; @@ -29,7 +32,7 @@ let }; startAt = mkOption { default = "hourly"; - type = types.str; # TODO systemd.time(7)'s calendar event + type = with types; nullOr str; # TODO systemd.time(7)'s calendar event }; snapshots = mkOption { default = { @@ -57,239 +60,165 @@ let }; imp = { - users.groups.backup.gid = genid "backup"; - users.users = {} - // { - root.openssh.authorizedKeys.keys = - map (plan: plan.dst.host.ssh.pubkey) - (filter isPullSrc (attrValues cfg.plans)) - ++ - map (plan: plan.src.host.ssh.pubkey) - (filter isPushDst (attrValues cfg.plans)) - ; - } - ; systemd.services = - flip mapAttrs' (filterAttrs (_:isPullDst) cfg.plans) (name: plan: { - name = "backup.${name}.pull"; - value = makePullService plan; - }) - // - flip mapAttrs' (filterAttrs (_:isPushSrc) cfg.plans) (name: plan: { - name = "backup.${name}.push"; - value = makePushService plan; - }) - ; - }; - - isPushSrc = plan: - plan.method == "push" && - plan.src.host.name == config.krebs.build.host.name; - - isPullSrc = plan: - plan.method == "pull" && - plan.src.host.name == config.krebs.build.host.name; - - isPushDst = plan: - plan.method == "push" && - plan.dst.host.name == config.krebs.build.host.name; - - isPullDst = plan: - plan.method == "pull" && - plan.dst.host.name == config.krebs.build.host.name; - - # TODO push destination needs this in the dst.user's PATH - service-path = [ - pkgs.coreutils - pkgs.gnused - pkgs.openssh - pkgs.rsync - pkgs.utillinux - ]; - - # TODO if there is plan.user, then use its privkey - makePushService = plan: assert isPushSrc plan; { - path = service-path; - serviceConfig = { - ExecStart = push plan; - Type = "oneshot"; - }; - startAt = plan.startAt; - }; + listToAttrs (map (plan: nameValuePair "backup.${plan.name}" { + # TODO if there is plan.user, then use its privkey + # TODO push destination users need a similar path + path = with pkgs; [ + coreutils + gnused + openssh + rsync + utillinux + ]; + serviceConfig = rec { + ExecStart = start plan; + SyslogIdentifier = ExecStart.name; + Type = "oneshot"; + }; + startAt = mkIf (plan.startAt != null) plan.startAt; + }) (filter (plan: build-host-is "pull" "dst" plan || + build-host-is "push" "src" plan) + enabled-plans)); - makePullService = plan: assert isPullDst plan; { - path = service-path; - serviceConfig = { - ExecStart = pull plan; - Type = "oneshot"; - }; - startAt = plan.startAt; + users.groups.backup.gid = genid "backup"; + users.users.root.openssh.authorizedKeys.keys = + map (plan: getAttr plan.method { + push = plan.src.host.ssh.pubkey; + pull = plan.dst.host.ssh.pubkey; + }) (filter (plan: build-host-is "pull" "src" plan || + build-host-is "push" "dst" plan) + enabled-plans); }; - push = plan: let - # We use writeDashBin and return the absolute path so systemd will produce - # nice names in the log, i.e. without the Nix store hash. - out = "${main}/bin/${main.name}"; - - main = writeDashBin "backup.${plan.name}.push" '' - set -efu - dst=${shell.escape plan.dst.path} - - mkdir -m 0700 -p "$dst" - exec flock -n "$dst" ${critical-section} - ''; - - critical-section = writeDash "backup.${plan.name}.push.critical-section" '' - # TODO check if there is a previous - set -efu - identity=${shell.escape plan.src.host.ssh.privkey.path} - src=${shell.escape plan.src.path} - dst_target=${shell.escape "root@${getFQDN plan.dst.host}"} - dst_path=${shell.escape plan.dst.path} - dst=$dst_target:$dst_path - - # Export NOW so runtime of rsync doesn't influence snapshot naming. - export NOW - NOW=$(date +%s) - - echo >&2 "update snapshot: current; $src -> $dst" - rsync >&2 \ - -aAXF --delete \ - -e "ssh -F /dev/null -i $identity" \ - --rsync-path ${shell.escape - "mkdir -m 0700 -p ${shell.escape plan.dst.path} && rsync"} \ - --link-dest="$dst_path/current" \ - "$src/" \ - "$dst/.partial" + enabled-plans = filter (getAttr "enable") (attrValues cfg.plans); - exec ssh -F /dev/null \ - -i "$identity" \ - "$dst_target" \ - -T \ - env NOW="$NOW" /bin/sh < ${remote-snapshot} - EOF - ''; + build-host-is = method: side: plan: + plan.method == method && + config.krebs.build.host.name == plan.${side}.host.name; - remote-snapshot = writeDash "backup.${plan.name}.push.remote-snapshot" '' - set -efu - dst=${shell.escape plan.dst.path} - - if test -e "$dst/current"; then - mv "$dst/current" "$dst/.previous" - fi - mv "$dst/.partial" "$dst/current" - rm -fR "$dst/.previous" - echo >&2 - - (${(take-snapshots plan).text}) - ''; - - in out; - - # TODO admit plan.dst.user and its ssh identity - pull = plan: let - # We use writeDashBin and return the absolute path so systemd will produce - # nice names in the log, i.e. without the Nix store hash. - out = "${main}/bin/${main.name}"; - - main = writeDashBin "backup.${plan.name}.pull" '' + start = plan: pkgs.writeDash "backup.${plan.name}" '' + set -efu + ${getAttr plan.method { + push = '' + identity=${shell.escape plan.src.host.ssh.privkey.path} + src_path=${shell.escape plan.src.path} + src=$src_path + dst_user=root + dst_host=$(${fastest-address plan.dst.host}) + dst_port=$(${network-ssh-port plan.dst.host "$dst_host"}) + dst_path=${shell.escape plan.dst.path} + dst=$dst_user@$dst_host:$dst_path + echo "update snapshot: current; $src -> $dst" >&2 + dst_shell() { + exec ssh -F /dev/null \ + -i "$identity" \ + ''${dst_port:+-p $dst_port} \ + "$dst_user@$dst_host" \ + -T "$with_dst_path_lock_script" + } + ''; + pull = '' + identity=${shell.escape plan.dst.host.ssh.privkey.path} + src_user=root + src_host=$(${fastest-address plan.src.host}) + src_port=$(${network-ssh-port plan.src.host "$src_host"}) + src_path=${shell.escape plan.src.path} + src=$src_user@$src_host:$src_path + dst_path=${shell.escape plan.dst.path} + dst=$dst_path + echo "update snapshot: current; $dst <- $src" >&2 + dst_shell() { + eval "$with_dst_path_lock_script" + } + ''; + }} + # Note that this only works because we trust date +%s to produce output + # that doesn't need quoting when used to generate a command string. + # TODO relax this requirement by selectively allowing to inject variables + # e.g.: ''${shell.quote "exec env NOW=''${shell.unquote "$NOW"} ..."} + with_dst_path_lock_script="exec env start_date=$(date +%s) "${shell.escape + "flock -n ${shell.escape plan.dst.path} /bin/sh" + } + rsync >&2 \ + -aAXF --delete \ + -e "ssh -F /dev/null -i $identity ''${dst_port:+-p $dst_port}" \ + --rsync-path ${shell.escape (concatStringsSep " && " [ + "mkdir -m 0700 -p ${shell.escape plan.dst.path}/current" + "exec flock -n ${shell.escape plan.dst.path} rsync" + ])} \ + --link-dest="$dst_path/current" \ + "$src/" \ + "$dst/.partial" + dst_shell < ${toFile "backup.${plan.name}.take-snapshots" '' set -efu - dst=${shell.escape plan.dst.path} - - mkdir -m 0700 -p "$dst" - exec flock -n "$dst" ${critical-section} - ''; + : $start_date - critical-section = writeDash "backup.${plan.name}.pull.critical-section" '' - # TODO check if there is a previous - set -efu - identity=${shell.escape plan.dst.host.ssh.privkey.path} - src=${shell.escape "root@${getFQDN plan.src.host}:${plan.src.path}"} dst=${shell.escape plan.dst.path} - # Export NOW so runtime of rsync doesn't influence snapshot naming. - export NOW - NOW=$(date +%s) - - echo >&2 "update snapshot: current; $dst <- $src" - mkdir -m 0700 -p ${shell.escape plan.dst.path} - rsync >&2 \ - -aAXF --delete \ - -e "ssh -F /dev/null -i $identity" \ - --link-dest="$dst/current" \ - "$src/" \ - "$dst/.partial" mv "$dst/current" "$dst/.previous" mv "$dst/.partial" "$dst/current" rm -fR "$dst/.previous" echo >&2 - exec ${take-snapshots plan} - ''; - in out; - - take-snapshots = plan: writeDash "backup.${plan.name}.take-snapshots" '' - set -efu - NOW=''${NOW-$(date +%s)} - dst=${shell.escape plan.dst.path} - - snapshot() {( - : $ns $format $retain - name=$(date --date="@$NOW" +"$format") - if ! test -e "$dst/$ns/$name"; then - echo >&2 "create snapshot: $ns/$name" - mkdir -m 0700 -p "$dst/$ns" - rsync >&2 \ - -aAXF --delete \ - --link-dest="$dst/current" \ - "$dst/current/" \ - "$dst/$ns/.partial.$name" - mv "$dst/$ns/.partial.$name" "$dst/$ns/$name" - echo >&2 - fi - case $retain in - ([0-9]*) - delete_from=$(($retain + 1)) - ls -r "$dst/$ns" \ - | sed -n "$delete_from,\$p" \ - | while read old_name; do - echo >&2 "delete snapshot: $ns/$old_name" - rm -fR "$dst/$ns/$old_name" - done - ;; - (ALL) - : - ;; - esac - )} - - ${concatStringsSep "\n" (mapAttrsToList (ns: { format, retain ? null, ... }: - toString (map shell.escape [ - "ns=${ns}" - "format=${format}" - "retain=${if retain == null then "ALL" else toString retain}" - "snapshot" - ])) - plan.snapshots)} + snapshot() {( + : $ns $format $retain + name=$(date --date="@$start_date" +"$format") + if ! test -e "$dst/$ns/$name"; then + echo >&2 "create snapshot: $ns/$name" + mkdir -m 0700 -p "$dst/$ns" + rsync >&2 \ + -aAXF --delete \ + --link-dest="$dst/current" \ + "$dst/current/" \ + "$dst/$ns/.partial.$name" + mv "$dst/$ns/.partial.$name" "$dst/$ns/$name" + echo >&2 + fi + case $retain in + ([0-9]*) + delete_from=$(($retain + 1)) + ls -r "$dst/$ns" \ + | sed -n "$delete_from,\$p" \ + | while read old_name; do + echo >&2 "delete snapshot: $ns/$old_name" + rm -fR "$dst/$ns/$old_name" + done + ;; + (ALL) + : + ;; + esac + )} + + ${concatStringsSep "\n" (mapAttrsToList (ns: { format, retain, ... }: + toString (map shell.escape [ + "ns=${ns}" + "format=${format}" + "retain=${if retain == null then "ALL" else toString retain}" + "snapshot" + ])) + plan.snapshots)} + ''} ''; - # TODO getFQDN: admit hosts in other domains - getFQDN = host: "${host.name}.${config.krebs.search-domain}"; - - writeDash = name: text: pkgs.writeScript name '' - #! ${pkgs.dash}/bin/dash - ${text} + # XXX Is one ping enough to determine fastest address? + fastest-address = host: '' + { ${pkgs.fping}/bin/fping </dev/null -a \ + ${concatMapStringsSep " " shell.escape + (mapAttrsToList (_: net: head net.aliases) host.nets)} \ + | ${pkgs.coreutils}/bin/head -1; } ''; - writeDashBin = name: text: pkgs.writeTextFile { - executable = true; - destination = "/bin/${name}"; - name = name; - text = '' - #! ${pkgs.dash}/bin/dash - ${text} - ''; - }; + # Note that we don't escape word on purpose, so we can deref shell vars. + # TODO type word + network-ssh-port = host: word: '' + case ${word} in + ${concatStringsSep ";;\n" (mapAttrsToList + (_: net: "(${head net.aliases}) echo ${toString net.ssh.port}") + host.nets)};; + esac + ''; in out # TODO ionice diff --git a/krebs/3modules/bepasty-server.nix b/krebs/3modules/bepasty-server.nix index e74841205..9e777a5ef 100644 --- a/krebs/3modules/bepasty-server.nix +++ b/krebs/3modules/bepasty-server.nix @@ -1,6 +1,6 @@ { config, lib, pkgs, ... }: -with lib; +with config.krebs.lib; let gunicorn = pkgs.pythonPackages.gunicorn; bepasty = pkgs.pythonPackages.bepasty-server; @@ -10,8 +10,8 @@ let out = { options.krebs.bepasty = api; - config = mkIf cfg.enable (mkMerge [ - (mkIf cfg.serveNginx nginx-imp) + config = lib.mkIf cfg.enable (lib.mkMerge [ + (lib.mkIf cfg.serveNginx nginx-imp) imp ]); }; diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 0da5dd38a..a1f446188 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -1,10 +1,8 @@ { config, lib, ... }: -with lib; +with config.krebs.lib; let - target = config.krebs.build // { user.name = "root"; }; - out = { # TODO deprecate krebs.build.host options.krebs.build.host = mkOption { @@ -17,12 +15,6 @@ let default = "/nix/var/nix/profiles/system"; }; - # TODO make krebs.build.target.host :: host - options.krebs.build.target = mkOption { - type = with types; nullOr str; - default = null; - }; - # TODO deprecate krebs.build.user options.krebs.build.user = mkOption { type = types.user; @@ -59,6 +51,7 @@ let source = config.krebs.build.source; target-user = maybeEnv "target_user" "root"; target-host = maybeEnv "target_host" config.krebs.build.host.name; + target-port = maybeEnv "target_port" "22"; target-path = maybeEnv "target_path" "/var/src"; out = '' #! /bin/sh @@ -70,7 +63,8 @@ let } echo ${shell.escape git-script} \ - | ssh ${shell.escape "${target-user}@${target-host}"} -T + | ssh -p ${shell.escape target-port} \ + ${shell.escape "${target-user}@${target-host}"} -T unset tmpdir trap ' @@ -101,6 +95,7 @@ let (attrNames file-specs)} \ --delete \ -vFrlptD \ + -e ${shell.escape "ssh -p ${target-port}"} \ ${shell.escape target-path}/ \ ${shell.escape "${target-user}@${target-host}:${target-path}"} ''; diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 74385a433..825cb3413 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -1,6 +1,6 @@ { config, pkgs, lib, ... }: -with lib; +with config.krebs.lib; let buildbot = pkgs.buildbot; buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' @@ -381,5 +381,5 @@ let in { options.krebs.buildbot.master = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; } diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 0e7796d8a..7705ac31c 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -1,6 +1,6 @@ { config, pkgs, lib, ... }: -with lib; +with config.krebs.lib; let buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' import os @@ -182,5 +182,5 @@ let in { options.krebs.buildbot.slave = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; } diff --git a/krebs/3modules/current.nix b/krebs/3modules/current.nix index 41941e289..9f63e33ac 100644 --- a/krebs/3modules/current.nix +++ b/krebs/3modules/current.nix @@ -1,13 +1,13 @@ { config, pkgs, lib, ... }: -with lib; +with config.krebs.lib; let cfg = config.krebs.current; out = { options.krebs.current = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; }; api = { diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index b1e0ef156..16a74e7c1 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -1,6 +1,6 @@ { config, lib, ... }: -with lib; +with config.krebs.lib; let cfg = config.krebs; @@ -9,9 +9,9 @@ let ./apt-cacher-ng.nix ./backup.nix ./bepasty-server.nix - ./build.nix ./buildbot/master.nix ./buildbot/slave.nix + ./build.nix ./current.nix ./exim-retiolum.nix ./exim-smarthost.nix @@ -20,17 +20,20 @@ let ./git.nix ./go.nix ./iptables.nix + ./lib.nix ./nginx.nix + ./nixpkgs.nix ./per-user.nix ./Reaktor.nix - ./retiolum-bootstrap.nix ./realwallpaper.nix + ./retiolum-bootstrap.nix ./retiolum.nix + ./setuid.nix ./tinc_graphs.nix ./urlwatch.nix ]; options.krebs = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; }; api = { @@ -81,13 +84,13 @@ let }; }; - imp = mkMerge [ - { krebs = import ./lass { inherit lib; }; } - { krebs = import ./makefu { inherit lib; }; } - { krebs = import ./miefda { inherit lib; }; } - { krebs = import ./mv { inherit lib; }; } - { krebs = import ./shared { inherit lib; }; } - { krebs = import ./tv { inherit lib; }; } + imp = lib.mkMerge [ + { krebs = import ./lass { inherit config lib; }; } + { krebs = import ./makefu { inherit config lib; }; } + { krebs = import ./miefda { inherit config lib; }; } + { krebs = import ./mv { inherit config lib; }; } + { krebs = import ./shared { inherit config lib; }; } + { krebs = import ./tv { inherit config lib; }; } { krebs.dns.providers = { de.krebsco = "zones"; @@ -156,7 +159,16 @@ let let inherit (config.krebs.build.host.ssh) privkey; in mkIf (privkey != null) (mkForce [privkey]); + # TODO use imports for merging services.openssh.knownHosts = + (let inherit (config.krebs.build.host.ssh) pubkey; in + optionalAttrs (pubkey != null) { + localhost = { + hostNames = ["localhost" "127.0.0.1" "::1"]; + publicKey = pubkey; + }; + }) + // # GitHub's IPv4 address range is 192.30.252.0/22 # Refs https://help.github.com/articles/what-ip-addresses-does-github-use-that-i-should-whitelist/ # 192.30.252.0/22 = 192.30.252.0-192.30.255.255 (1024 addresses) @@ -180,7 +192,6 @@ let (mapAttrsToList (net-name: net: let - aliases = shorts ++ longs; longs = net.aliases; shorts = map (removeSuffix ".${cfg.search-domain}") @@ -191,7 +202,7 @@ let then "[${a}]:${toString net.ssh.port}" else a; in - aliases ++ map add-port net.addrs) + map add-port (shorts ++ longs ++ net.addrs)) host.nets); publicKey = host.ssh.pubkey; diff --git a/krebs/3modules/exim-retiolum.nix b/krebs/3modules/exim-retiolum.nix index ea012c38c..1722eef1f 100644 --- a/krebs/3modules/exim-retiolum.nix +++ b/krebs/3modules/exim-retiolum.nix @@ -1,12 +1,12 @@ { config, pkgs, lib, ... }: -with lib; +with config.krebs.lib; let cfg = config.krebs.exim-retiolum; out = { options.krebs.exim-retiolum = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; }; api = { diff --git a/krebs/3modules/exim-smarthost.nix b/krebs/3modules/exim-smarthost.nix index a564430ea..267ee2900 100644 --- a/krebs/3modules/exim-smarthost.nix +++ b/krebs/3modules/exim-smarthost.nix @@ -1,13 +1,12 @@ { config, pkgs, lib, ... }: -with builtins; -with lib; +with config.krebs.lib; let cfg = config.krebs.exim-smarthost; out = { options.krebs.exim-smarthost = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; }; api = { diff --git a/krebs/3modules/fetchWallpaper.nix b/krebs/3modules/fetchWallpaper.nix index f320c7505..53fe0839d 100644 --- a/krebs/3modules/fetchWallpaper.nix +++ b/krebs/3modules/fetchWallpaper.nix @@ -1,21 +1,17 @@ { config, lib, pkgs, ... }: -with lib; +with config.krebs.lib; let cfg = config.krebs.fetchWallpaper; out = { options.krebs.fetchWallpaper = api; - config = mkIf cfg.enable imp; + config = lib.mkIf cfg.enable imp; }; api = { enabl |