diff options
36 files changed, 907 insertions, 433 deletions
@@ -1,8 +1,9 @@ # # usage: -# make system=foo -# make systems='foo bar' -# make eval system=foo get=config.networking.extraHosts [filter=json] +# make infest system=foo [target=bar] +# make [deploy] system=foo [target=bar] +# make [deploy] systems='foo bar' +# make eval get=tv.wu.config.time.timeZone [filter=json] # .ONESHELL: @@ -10,20 +11,20 @@ ifdef systems $(systems): + @ + unset target parallel \ --line-buffer \ -j0 \ --no-notice \ --tagstring {} \ - -q make systems= system={} ::: $(systems) + -q make -s systems= system={} ::: $(systems) else ifdef system -.PHONY: deploy -deploy:;@ - make -s eval system=$(system) get=config.krebs.build.script filter=json | sh - -.PHONY: infest -infest:;@ - make -s eval system=$(system) get=config.krebs.build.infest filter=json | sh +.PHONY: deploy infest +deploy infest:;@ + export get=krebs.$@ + export filter=json + make -s eval | sh .PHONY: eval eval: @@ -40,8 +41,11 @@ endif --eval \ -A "$$get" \ '<stockholm>' \ - --argstr user-name "$$LOGNAME" \ - --argstr system-name "$$system" \ + --argstr current-date "$$(date -Is)" \ + --argstr current-host-name "$$HOSTNAME" \ + --argstr current-user-name "$$LOGNAME" \ + $${system+--argstr system "$$system"} \ + $${target+--argstr target "$$target"} \ | filter else $(error unbound variable: system[s]) diff --git a/default.nix b/default.nix index 875f0d5b4..c6a635c29 100644 --- a/default.nix +++ b/default.nix @@ -1,26 +1,80 @@ -{ user-name, system-name }: +{ current-date +, current-host-name +, current-user-name +}: let + lib = import <nixpkgs/lib>; - eval = import <nixpkgs/nixos/lib/eval-config.nix> { + krebs-modules-path = ./krebs/3modules; + krebs-pkgs-path = ./krebs/5pkgs; + user-modules-path = ./. + "/${current-user-name}/3modules"; + user-pkgs-path = ./. + "/${current-user-name}/5pkgs"; + + out = + lib.mapAttrs (_: builtins.getAttr "main") + (lib.filterAttrs (_: builtins.hasAttr "main") + (lib.mapAttrs + (k: v: + if lib.hasPrefix "." k || v != "directory" then + {} + else if builtins.pathExists (./. + "/${k}/default.nix") then + { main = import (./. + "/${k}"); } + else if builtins.pathExists (./. + "/${k}/1systems") then + { main = mk-namespace (./. + "/${k}"); } + else + {}) + (builtins.readDir ./.))); + + eval = path: import <nixpkgs/nixos/lib/eval-config.nix> { system = builtins.currentSystem; - modules = map (p: ./. + "/${p}") [ - "${user-name}/1systems/${system-name}.nix" - "${user-name}/3modules" - "krebs/3modules" + modules = [ + path + krebs-modules-path + user-modules-path ] ++ [ - ({ lib, pkgs, ... }: { + ({ config, lib, pkgs, ... }@args: { _module.args.pkgs = - (import ./krebs/5pkgs { inherit lib pkgs; }) // - (import (./. + "/${user-name}/5pkgs") { inherit lib pkgs; }); + (import krebs-pkgs-path args) // + (import user-pkgs-path args); }) ]; }; -in + mk-namespace = path: mapNixDir mk-system (path + "/1systems"); + + mk-system = path: rec { + inherit (eval path) config options; + system = config.system.build.toplevel; + fetch = import ./krebs/0tools/fetch.nix { inherit config lib; }; + }; + + mapNixDir = f: path: lib.mapAttrs (_: f) (nixDir path); + + nixDir = path: + builtins.listToAttrs + (catMaybes + (lib.mapAttrsToList + (k: v: { + directory = + let p = path + "/${k}/default.nix"; in + if builtins.pathExists p + then Just (lib.nameValuePair k p) + else Nothing; + regular = + let p = path + "/${k}"; in + if lib.hasSuffix ".nix" p + then Just (lib.nameValuePair (lib.removeSuffix ".nix" k) p) + else Nothing; + }.${v} or Nothing) + (builtins.readDir path))); -{ - inherit (eval) config options; + # TODO move to lib + Just = x: { type = "maybe"; value = x; }; + Nothing = { type = "maybe"; }; + isMaybe = x: builtins.typeOf x == "set" && x.type or false == "maybe"; + isJust = x: isMaybe x && builtins.hasAttr "value" x; + fromJust = x: assert isJust x; x.value; + catMaybes = xs: map fromJust (builtins.filter isJust xs); - system = eval.config.system.build.toplevel; -} +in out diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index fce24fa63..82089a660 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -2,7 +2,6 @@ let - kpkgs = import ../5pkgs { inherit pkgs; inherit lib; }; inherit (lib) mkIf @@ -63,13 +62,20 @@ let configuration appended to the default or overridden configuration ''; }; - - ReaktorPkg = mkOption { - default = kpkgs.Reaktor; + extraEnviron = mkOption { + default = {}; + type = types.attrsOf types.str; description = '' - the Reaktor pkg to use. + Environment to be provided to the service, can be: + REAKTOR_HOST + REAKTOR_PORT + REAKTOR_STATEDIR + REAKTOR_CHANNELS + + debug and nickname can be set separately via the Reaktor api ''; }; + debug = mkOption { default = false; description = '' @@ -80,7 +86,6 @@ let imp = { # for reaktor get-config - environment.systemPackages = [ cfg.ReaktorPkg ]; users.extraUsers = singleton { name = "Reaktor"; # uid = config.ids.uids.Reaktor; @@ -98,7 +103,7 @@ let systemd.services.Reaktor = { path = with pkgs; [ utillinux #flock for tell_on-join - # git # for nag + git # for nag python # for caps ]; description = "Reaktor IRC Bot"; @@ -108,17 +113,17 @@ let GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; REAKTOR_NICKNAME = cfg.nickname; REAKTOR_DEBUG = (if cfg.debug then "True" else "False"); - }; + } // cfg.extraEnviron; serviceConfig= { ExecStartPre = pkgs.writeScript "Reaktor-init" '' #! /bin/sh ${if (isString cfg.overrideConfig) then ''cp ${ReaktorConfig} /tmp/config.py'' else - ''(${cfg.ReaktorPkg}/bin/reaktor get-config;cat "${ReaktorConfig}" ) > /tmp/config.py'' + ''(${pkgs.Reaktor}/bin/reaktor get-config;cat "${ReaktorConfig}" ) > /tmp/config.py'' } ''; - ExecStart = "${cfg.ReaktorPkg}/bin/reaktor run /tmp/config.py"; + ExecStart = "${pkgs.Reaktor}/bin/reaktor run /tmp/config.py"; PrivateTmp = "true"; User = "Reaktor"; Restart = "on-abort"; diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix new file mode 100644 index 000000000..57495ea69 --- /dev/null +++ b/krebs/3modules/build.nix @@ -0,0 +1,72 @@ +{ config, lib, ... }: + +with import ../4lib { inherit lib; }; + +let + target = config.krebs.build // { user.name = "root"; }; + + out = { + # TODO deprecate krebs.build.host + options.krebs.build.host = mkOption { + type = types.host; + }; + + # TODO make krebs.build.profile shell safe + options.krebs.build.profile = mkOption { + type = types.str; + default = "/nix/var/nix/profiles/system"; + }; + + # TODO make krebs.build.target.host :: host + options.krebs.build.target = mkOption { + type = with types; nullOr str; + default = null; + }; + + # TODO deprecate krebs.build.user + options.krebs.build.user = mkOption { + type = types.user; + }; + + options.krebs.build.source.dir = mkOption { + type = types.attrsOf (types.submodule ({ config, ... }: { + options = { + host = mkOption { + type = types.host; + }; + path = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + url = mkOption { + type = types.str; + default = "file://${config.host.name}${config.path}"; + }; + }; + })); + default = {}; + }; + + options.krebs.build.source.git = mkOption { + type = with types; attrsOf (submodule ({ config, ... }: { + options = { + url = mkOption { + type = types.str; # TODO must be shell safe + }; + rev = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + }; + })); + default = {}; + }; + }; + +in out diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 0ffdec5f8..de265b914 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -6,6 +6,7 @@ let out = { imports = [ + ./build.nix ./exim-retiolum.nix ./exim-smarthost.nix ./github-hosts-sync.nix @@ -22,225 +23,6 @@ let api = { enable = mkEnableOption "krebs"; - build = mkOption { - type = types.submodule ({ config, ... }: { - options = { - target = mkOption { - type = with types; nullOr str; - default = null; - }; - deps = mkOption { - type = with types; attrsOf (submodule { - options = { - url = mkOption { - type = str; - }; - rev = mkOption { - type = nullOr str; - default = null; - }; - }; - }); - default = {}; - }; - script = mkOption { - type = types.str; - default = '' - #! /bin/sh - set -efux - - target=${escapeShellArg cfg.build.target} - - push(){( - src=$1/ - dst=$target:$2 - rsync \ - --exclude .git \ - --exclude .graveyard \ - --exclude old \ - --rsync-path="mkdir -p \"$2\" && rsync" \ - --delete-excluded \ - -vrLptgoD \ - "$src" "$dst" - )} - - ${concatStrings (mapAttrsToList (name: { url, rev, ... }: - optionalString (rev == null) '' - push ${toString (map escapeShellArg [ - "${url}" - "/root/src/${name}" - ])} - '') config.deps)} - - exec ssh -S none "$target" /bin/sh <<\EOF - set -efux - fetch(){( - url=$1 - rev=$2 - dst=$3 - mkdir -p "$dst" - cd "$dst" - if ! test -e .git; then - git init - fi - if ! cur_url=$(git config remote.origin.url 2>/dev/null); then - git remote add origin "$url" - elif test "$cur_url" != "$url"; then - git remote set-url origin "$url" - fi - if test "$(git rev-parse --verify HEAD 2>/dev/null)" != "$rev"; then - git fetch origin - git checkout "$rev" -- . - git checkout -q "$rev" - git submodule init - git submodule update - fi - git clean -dxf - )} - - ${concatStrings (mapAttrsToList (name: { url, rev, ... }: - optionalString (rev != null) '' - fetch ${toString (map escapeShellArg [ - url - rev - "/root/src/${name}" - ])} - '') config.deps)} - - echo build system... - profile=/nix/var/nix/profiles/system - NIX_PATH=/root/src \ - nix-env \ - -Q \ - -p "$profile" \ - -f '<stockholm>' \ - --set \ - -A system \ - --argstr user-name ${escapeShellArg cfg.build.user.name} \ - --argstr system-name ${escapeShellArg cfg.build.host.name} - - exec "$profile"/bin/switch-to-configuration switch - EOF - - ''; - }; - infest = mkOption { - type = types.str; - default = '' - #! /bin/sh - set -efux - - target=${escapeShellArg cfg.build.target} - - push(){( - src=$1/ - dst=$target:/mnt$2 - rsync \ - --exclude .git \ - --exclude .graveyard \ - --exclude old \ - --rsync-path="mkdir -p \"/mnt$2\" && rsync" \ - --delete-excluded \ - -vrLptgoD \ - "$src" "$dst" - )} - - cat krebs/4lib/infest/1prepare | ssh "$target" - cat krebs/4lib/infest/2install-nix | ssh "$target" - - ${concatStrings (mapAttrsToList (name: { url, rev, ... }: - optionalString (rev == null) '' - push ${toString (map escapeShellArg [ - "${url}" - "/root/src/${name}" - ])} - '') config.deps)} - - ssh -S none "$target" /bin/sh <<\EOF - set -efux - - fetch(){( - url=$1 - rev=$2 - dst=$3 - mkdir -p "$dst" - cd "$dst" - if ! test -e .git; then - git init - fi - if ! cur_url=$(git config remote.origin.url 2>/dev/null); then - git remote add origin "$url" - elif test "$cur_url" != "$url"; then - git remote set-url origin "$url" - fi - if test "$(git rev-parse --verify HEAD 2>/dev/null)" != "$rev"; then - git fetch origin - git checkout "$rev" -- . - git checkout -q "$rev" - git submodule init - git submodule update - fi - git clean -dxf - )} - - ${concatStrings (mapAttrsToList (name: { url, rev, ... }: - optionalString (rev != null) '' - fetch ${toString (map escapeShellArg [ - url - rev - "/mnt/root/src/${name}" - ])} - '') config.deps)} - - export PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:$PATH - - sed < "$(type -p nixos-install)" > nixos-install ' - /^echo "building the system configuration..."/,/--set -A system/{ - s/.*/# &/ - s@.*--set -A system.*@&\n${concatStringsSep " " [ - "NIX_PATH=/mnt/root/src/" - "nix-env" - "-Q" - "-p /nix/var/nix/profiles/system" - "-f \"<stockholm>\"" - "--set" - "-A system" - "--argstr user-name ${escapeShellArg cfg.build.user.name}" - "--argstr system-name ${escapeShellArg cfg.build.host.name}" - ]}@ - } - ' - - sed -i 's/^nixpkgs=.*$/#&/' nixos-install - - - chmod +x nixos-install - - echo {} > /root/dummy.nix - - echo build system... - profile=/nix/var/nix/profiles/system - NIXOS_CONFIG=/root/dummy.nix \ - ./nixos-install -I /root/src/ - #nl -bp nixos-install - - EOF - - cat krebs/4lib/infest/4finalize | ssh "$target" - ''; - }; - host = mkOption { - type = types.host; - }; - user = mkOption { - type = types.user; - }; - }; - }); - # Define defaul value, so unset values of the submodule get reported. - default = {}; - }; - dns = { providers = mkOption { # TODO with types; tree dns.label dns.provider, so we can merge. @@ -302,13 +84,16 @@ let mapAttrsToList (hostname: host: mapAttrsToList (netname: net: let - aliases = toString (unique (longs ++ shorts)); + aliases = longs ++ shorts; providers = dns.split-by-provider net.aliases cfg.dns.providers; longs = providers.hosts; - shorts = map (removeSuffix ".${cfg.search-domain}") longs; + shorts = + map (removeSuffix ".${cfg.search-domain}") + (filter (hasSuffix ".${cfg.search-domain}") + longs); in - map (addr: "${addr} ${aliases}") net.addrs - ) host.nets + map (addr: "${addr} ${toString aliases}") net.addrs + ) (filterAttrs (name: host: host.aliases != []) host.nets) ) cfg.hosts )); @@ -318,6 +103,36 @@ let ([cfg.zone-head-config] ++ combined-hosts) ; combined-hosts = (mapAttrsToList (name: value: value.extraZones) cfg.hosts ); in lib.mapAttrs' (name: value: nameValuePair (("zones/" + name)) ({ text=value; })) all-zones; + + services.openssh.hostKeys = + let inherit (config.krebs.build.host.ssh) privkey; in + mkIf (privkey != null) (mkForce [privkey]); + + services.openssh.knownHosts = + mapAttrs + (name: host: { + hostNames = + concatLists + (mapAttrsToList + (net-name: net: + let + aliases = shorts ++ longs; + longs = net.aliases; + shorts = + map (removeSuffix ".${cfg.search-domain}") + (filter (hasSuffix ".${cfg.search-domain}") + longs); + add-port = a: + if net.ssh.port != null + then "[${a}]:${toString net.ssh.port}" + else a; + in + aliases ++ map add-port net.addrs) + host.nets); + + publicKey = host.ssh.pubkey; + }) + (filterAttrs (_: host: host.ssh.pubkey != null) cfg.hosts); } ]; @@ -537,8 +352,8 @@ let extraZones = { "krebsco.de" = '' - mediengewitter IN A ${elemAt nets.internet.addrs4 0} - flap IN A ${elemAt nets.internet.addrs4 0}''; + mediengewitter IN A ${head nets.internet.addrs4} + flap IN A ${head nets.internet.addrs4}''; }; nets = { internet = { @@ -575,14 +390,13 @@ let IN MX 10 mx42 euer IN MX 1 aspmx.l.google.com. io IN NS pigstarter.krebsco.de. - euer IN A ${elemAt nets.internet.addrs4 0} - pigstarter IN A ${elemAt nets.internet.addrs4 0} - conf IN A ${elemAt nets.internet.addrs4 0} - gold IN A ${elemAt nets.internet.addrs4 0} - graph IN A ${elemAt nets.internet.addrs4 0} - tinc IN A ${elemAt nets.internet.addrs4 0} - boot IN A ${elemAt nets.internet.addrs4 0} - mx42 IN A ${elemAt nets.internet.addrs4 0}''; + pigstarter IN A ${head nets.internet.addrs4} + conf IN A ${head nets.internet.addrs4} + gold IN A ${head nets.internet.addrs4} + graph IN A ${head nets.internet.addrs4} + tinc IN A ${head nets.internet.addrs4} + boot IN A ${head nets.internet.addrs4} + mx42 IN A ${head nets.internet.addrs4}''; }; nets = { internet = { @@ -611,15 +425,56 @@ let }; }; }; + wry = rec { + cores = 1; + dc = "makefu"; #dc = "cac"; + extraZones = { + "krebsco.de" = '' + wry IN A ${head nets.internet.addrs4} + ''; + }; + nets = rec { + internet = { + addrs4 = ["162.219.7.216"]; + aliases = [ + "wry.internet" + ]; + }; + retiolum = { + via = internet; + addrs4 = ["10.243.29.169"]; + addrs6 = ["42:6e1e:cc8a:7cef:827:f938:8c64:baad"]; + aliases = [ + "wry.retiolum" + ]; + tinc.pubkey = '' + -----BEGIN RSA PUBLIC KEY----- + MIICCgKCAgEAvmCBVNKT/Su4v9nl/Nm3STPo5QxWPg7xEkzIs3Oh39BS8+r6/7UQ + rebib7mczb+ebZd+Rg2yFoGrWO8cmM0VcLy5bYRMK7in8XroLEjWecNNM4TRfNR4 + e53+LhcPdkxo0A3/D+yiut+A2Mkqe+4VXDm/JhAiAYkZTn7jUtj00Atrc7CWW1gN + sP3jIgv4+CGftdSYOB4dm699B7OD9XDLci2kOaFqFl4cjDYUok03G0AduUlRx10v + CKbKOTIdm8C36A902/3ms+Hyzkruu+VagGIZuPSwqXHJPCu7Ju+jarKQstMmpQi0 + PubweWDL0o/Dfz2qT3DuL4xDecIvGE6kv3m41hHJYiK+2/azTSehyPFbsVbL7w0V + LgKN3usnZNcpTsBWxRGT7nMFSnX2FLDu7d9OfCuaXYxHVFLZaNrpccOq8NF/7Hbk + DDW81W7CvLyJDlp0WLnAawSOGTUTPoYv/2wAapJ89i8QGCueGvEc6o2EcnBVMFEW + ejWTQzyD816f4RsplnrRqLVlIMbr9Q/n5TvlgjjhX7IMEfMy4+7qLGRQkNbFzgwK + jxNG2fFSCjOEQitm0gAtx7QRIyvYr6c7/xiHz4AwxYzBmvQsL/OK57NO4+Krwgj5 + Vk8TQ2jGO7J4bB38zaxK+Lrtfl8i1AK1171JqFMhOc34JSJ7T4LWDMECAwEAAQ== + -----END RSA PUBLIC KEY----- + ''; + }; + }; + }; gum = rec { cores = 1; dc = "online.net"; #root-server extraZones = { "krebsco.de" = '' - omo IN A ${elemAt nets.internet.addrs4 0} - gum IN A ${elemAt nets.internet.addrs4 0} - paste IN A ${elemAt nets.internet.addrs4 0}''; + omo IN A ${head nets.internet.addrs4} + euer IN A ${head nets.internet.addrs4} + gum IN A ${head nets.internet.addrs4} + paste IN A ${head nets.internet.addrs4}''; }; nets = { internet = { @@ -679,6 +534,7 @@ let "cgit.cd.viljetic.de" "cd.krebsco.de" ]; + ssh.port = 11423; }; retiolum = { via = internet; @@ -705,13 +561,15 @@ let ''; }; }; + ssh.privkey.path = <secrets/ssh.id_ed25519>; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOd/HqZIO9Trn3eycl23GZAz21HQCISaVNfNyaLSQvJ6"; }; - mkdir = { + mkdir = rec { cores = 1; dc = "tv"; #dc = "cac"; nets = rec { internet = { - addrs4 = ["162.248.167.241"]; + addrs4 = ["104.233.84.215"]; aliases = [ "mkdir.internet" ]; @@ -736,6 +594,35 @@ let ''; |