diff options
54 files changed, 1418 insertions, 1120 deletions
@@ -1,5 +1,32 @@ -ifndef system -$(error unbound variable: system) +stockholm ?= . + +ifndef nixos-config +$(if $(system),,$(error unbound variable: system)) +nixos-config = ./$(LOGNAME)/1systems/$(system).nix +endif + +# target = [target_user@]target_host[:target_port][/target_path] +ifdef target +_target_user != echo $(target) | sed -n 's/@.*//p' +_target_path != echo $(target) | sed -n 's/^[^/]*//p' +_target_port != echo $(target) | sed -En 's|^.*:([^/]*)(/.*)?$$|\1|p' +_target_host != echo $(target) | sed -En 's/^(.*@)?([^:/]*).*/\2/p' +ifneq ($(_target_host),) +$(if $(target_host),$(error cannot define both, target_host and host in target)) +target_host ?= $(_target_host) +endif +ifneq ($(_target_user),) +$(if $(target_user),$(error cannot define both, target_user and user in target)) +target_user ?= $(_target_user) +endif +ifneq ($(_target_port),) +$(if $(target_port),$(error cannot define both, target_port and port in target)) +target_port ?= $(_target_port) +endif +ifneq ($(_target_path),) +$(if $(target_path),$(error cannot define both, target_path and path in target)) +target_path ?= $(_target_path) +endif endif export target_host ?= $(system) @@ -7,29 +34,37 @@ export target_user ?= root export target_port ?= 22 export target_path ?= /var/src +$(if $(target_host),,$(error unbound variable: target_host)) +$(if $(target_user),,$(error unbound variable: target_user)) +$(if $(target_port),,$(error unbound variable: target_port)) +$(if $(target_path),,$(error unbound variable: target_path)) + evaluate = \ nix-instantiate \ --eval \ --readonly-mode \ --show-trace \ - -I nixos-config=./$(LOGNAME)/1systems/$(system).nix \ - -I stockholm=. \ - $(1) + -I nixos-config=$(nixos-config) \ + -I stockholm=$(stockholm) \ + -E '{ eval, f }: f eval' \ + --arg eval 'import ./.' \ + --arg f "eval@{ config, ... }: $(1)" execute = \ - result=$$($(call evaluate,-A config.krebs.build.$(1) --json)) && \ + result=$$($(call evaluate,config.krebs.build.$(1))) && \ script=$$(echo "$$result" | jq -r .) && \ - echo "$$script" | sh + echo "$$script" | PS5=% sh # usage: make deploy system=foo [target_host=bar] +deploy: ssh ?= ssh deploy: $(call execute,populate) - ssh $(target_user)@$(target_host) -p $(target_port) \ + $(ssh) $(target_user)@$(target_host) -p $(target_port) \ nixos-rebuild switch --show-trace -I $(target_path) # usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name -eval eval.:;@$(call evaluate) -eval.%:;@$(call evaluate,-A $*) +eval eval.:;@$(call evaluate,$${expr-eval}) +eval.%:;@$(call evaluate,$*) # usage: make install system=foo [target_host=bar] install: ssh ?= ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null @@ -41,3 +76,21 @@ install: $(ssh) $(target_user)@$(target_host) -p $(target_port) \ env NIXOS_CONFIG=$(target_path)/nixos-config \ nixos-install + +# usage: make test system=foo [target=bar] [method={eval,build}] +method ?= eval +ifeq ($(method),build) +test: command = nix-build --no-out-link +else +ifeq ($(method),eval) +test: command ?= nix-instantiate --eval --json --readonly-mode --strict +else +$(error bad method: $(method)) +endif +endif +test: ssh ?= ssh +test: + $(call execute,populate) + $(ssh) $(target_user)@$(target_host) -p $(target_port) \ + $(command) --show-trace -I $(target_path) \ + -A config.system.build.toplevel $(target_path)/stockholm diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix index 0aa86dec9..97082f56a 100644 --- a/krebs/3modules/backup.nix +++ b/krebs/3modules/backup.nix @@ -117,6 +117,14 @@ let "$dst_user@$dst_host" \ -T "$with_dst_path_lock_script" } + rsh="ssh -F /dev/null -i $identity ''${dst_port:+-p $dst_port}" + local_rsync() { + rsync "$@" + } + remote_rsync=${shell.escape (concatStringsSep " && " [ + "mkdir -m 0700 -p ${shell.escape plan.dst.path}/current" + "exec flock -n ${shell.escape plan.dst.path} rsync" + ])} ''; pull = '' identity=${shell.escape plan.dst.host.ssh.privkey.path} @@ -131,6 +139,12 @@ let dst_shell() { eval "$with_dst_path_lock_script" } + rsh="ssh -F /dev/null -i $identity ''${src_port:+-p $src_port}" + local_rsync() { + mkdir -m 0700 -p ${shell.escape plan.dst.path}/current + flock -n ${shell.escape plan.dst.path} rsync "$@" + } + remote_rsync=rsync ''; }} # Note that this only works because we trust date +%s to produce output @@ -140,13 +154,10 @@ let with_dst_path_lock_script="exec env start_date=$(date +%s) "${shell.escape "flock -n ${shell.escape plan.dst.path} /bin/sh" } - rsync >&2 \ + local_rsync >&2 \ -aAXF --delete \ - -e "ssh -F /dev/null -i $identity ''${dst_port:+-p $dst_port}" \ - --rsync-path ${shell.escape (concatStringsSep " && " [ - "mkdir -m 0700 -p ${shell.escape plan.dst.path}/current" - "exec flock -n ${shell.escape plan.dst.path} rsync" - ])} \ + --rsh="$rsh" \ + --rsync-path="$remote_rsync" \ --link-dest="$dst_path/current" \ "$src/" \ "$dst/.partial" diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index a1f446188..d4c6b08df 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -20,35 +20,19 @@ let type = types.user; }; - options.krebs.build.source = let - raw = types.either types.str types.path; - url = types.submodule { + options.krebs.build.source = mkOption { + type = with types; attrsOf (either str (submodule { options = { - url = mkOption { - type = types.str; - }; - rev = mkOption { - type = types.str; - }; - dev = mkOption { - type = types.str; - }; + url = str; + rev = str; }; - }; - in mkOption { - type = types.attrsOf (types.either types.str url); - apply = let f = mapAttrs (_: value: { - string = value; - path = toString value; - set = f value; - }.${typeOf value}); in f; + })); default = {}; }; options.krebs.build.populate = mkOption { type = types.str; default = let - source = config.krebs.build.source; target-user = maybeEnv "target_user" "root"; target-host = maybeEnv "target_host" config.krebs.build.host.name; target-port = maybeEnv "target_port" "22"; @@ -57,13 +41,16 @@ let #! /bin/sh set -eu + ssh=''${ssh-ssh} + verbose() { - printf '+%s\n' "$(printf ' %q' "$@")" >&2 + printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2 "$@" } - echo ${shell.escape git-script} \ - | ssh -p ${shell.escape target-port} \ + { printf 'PS5=%q%q\n' @ "$PS5" + echo ${shell.escape git-script} + } | verbose $ssh -p ${shell.escape target-port} \ ${shell.escape "${target-user}@${target-host}"} -T unset tmpdir @@ -75,61 +62,34 @@ let tmpdir=$(mktemp -dt stockholm.XXXXXXXX) chmod 0755 "$tmpdir" - ${concatStringsSep "\n" - (mapAttrsToList - (name: spec: let dst = removePrefix "symlink:" (get-url spec); in - "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") - symlink-specs)} + ${concatStringsSep "\n" (mapAttrsToList (name: symlink: '' + verbose ln -s ${shell.escape symlink.target} \ + "$tmpdir"/${shell.escape name} + '') source-by-method.symlink)} verbose proot \ - -b $tmpdir:${shell.escape target-path} \ - ${concatStringsSep " \\\n " - (mapAttrsToList - (name: spec: - "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}") - file-specs)} \ + -b "$tmpdir":${shell.escape target-path} \ + ${concatStringsSep " \\\n " (mapAttrsToList (name: file: + "-b ${shell.escape "${file.path}:${target-path}/${name}"}" + ) source-by-method.file)} \ rsync \ -f ${shell.escape "P /*"} \ - ${concatMapStringsSep " \\\n " - (name: "-f ${shell.escape "R /${name}"}") - (attrNames file-specs)} \ + ${concatMapStringsSep " \\\n " (name: + "-f ${shell.escape "R /${name}"}" + ) (attrNames source-by-method.file)} \ --delete \ -vFrlptD \ - -e ${shell.escape "ssh -p ${target-port}"} \ + -e "$ssh -p ${shell.escape target-port}" \ ${shell.escape target-path}/ \ ${shell.escape "${target-user}@${target-host}:${target-path}"} ''; - get-schema = uri: - if substring 0 1 uri == "/" - then "file" - else head (splitString ":" uri); - - has-schema = schema: uri: get-schema uri == schema; - - get-url = spec: { - string = spec; - path = toString spec; - set = get-url spec.url; - }.${typeOf spec}; - - git-specs = - filterAttrs (_: spec: has-schema "https" (get-url spec)) source // - filterAttrs (_: spec: has-schema "http" (get-url spec)) source // - filterAttrs (_: spec: has-schema "git" (get-url spec)) source; - - file-specs = - filterAttrs (_: spec: has-schema "file" (get-url spec)) source; - - symlink-specs = - filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source; - git-script = '' #! /bin/sh set -efu verbose() { - printf '+%s\n' "$(printf ' %q' "$@")" >&2 + printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2 "$@" } @@ -156,26 +116,48 @@ let if ! test "$(git log --format=%H -1)" = "$hash"; then git fetch origin git checkout "$hash" -- "$dst_dir" - git checkout "$hash" + git checkout -f "$hash" fi git clean -dxf )} - ${concatStringsSep "\n" - (mapAttrsToList - (name: spec: toString (map shell.escape [ - "verbose" - "fetch_git" - "${target-path}/${name}" - spec.url - spec.rev - ])) - git-specs)} + ${concatStringsSep "\n" (mapAttrsToList (name: git: '' + verbose fetch_git ${concatMapStringsSep " " shell.escape [ + "${target-path}/${name}" + git.url + git.rev + ]} + '') source-by-method.git)} ''; in out; }; }; + source-by-method = let + known-methods = ["git" "file" "symlink"]; + in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme { + git = source-by-scheme.http or {} // + source-by-scheme.https or {}; + }; + + source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out { + ${v.scheme}.${k} = v; + }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source); + + normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) { + path = f (toString x); + string = f { + url = if substring 0 1 x == "/" then "file://${x}" else x; + }; + set = let scheme = head (splitString ":" x.url); in recursiveUpdate x { + inherit scheme; + } // { + symlink.target = removePrefix "symlink:" x.url; + file.path = # TODO file://host/... + assert hasPrefix "file:///" x.url; + removePrefix "file://" x.url; + }.${scheme} or {}; + }; in f) config.krebs.build.source; in out diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index c06f3754e..186469e97 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -28,6 +28,7 @@ let ./realwallpaper.nix ./retiolum-bootstrap.nix ./retiolum.nix + ./secret.nix ./setuid.nix ./tinc_graphs.nix ./urlwatch.nix @@ -42,9 +43,7 @@ let dns = { providers = mkOption { - # TODO with types; tree dns.label dns.provider, so we can merge. - # Currently providers can only be merged if aliases occur just once. - type = with types; attrsOf unspecified; + type = with types; attrsOf str; }; }; @@ -94,7 +93,7 @@ let { krebs = import ./tv { inherit config lib; }; } { krebs.dns.providers = { - de.krebsco = "zones"; + "krebsco.de" = "zones"; gg23 = "hosts"; shack = "hosts"; i = "hosts"; @@ -103,13 +102,27 @@ let retiolum = "hosts"; }; - networking.extraHosts = concatStringsSep "\n" (flatten ( + krebs.users = { + krebs = { + home = "/krebs"; + mail = "spam@krebsco.de"; + }; + root = { + home = "/root"; + pubkey = config.krebs.build.host.ssh.pubkey; + uid = 0; + }; + }; + + networking.extraHosts = let + domains = attrNames (filterAttrs (_: eq "hosts") cfg.dns.providers); + check = hostname: any (domain: hasSuffix ".${domain}" hostname) domains; + in concatStringsSep "\n" (flatten ( mapAttrsToList (hostname: host: mapAttrsToList (netname: net: let aliases = longs ++ shorts; - providers = dns.split-by-provider net.aliases cfg.dns.providers; - longs = providers.hosts; + longs = filter check net.aliases; shorts = let s = ".${cfg.search-domain}"; in map (removeSuffix s) (filter (hasSuffix s) longs); in @@ -130,12 +143,11 @@ let { text=(stripEmptyLines value); }) all-zones; krebs.exim-smarthost.internet-aliases = let - format = from: to: + format = from: to: { + inherit from; # TODO assert is-retiolum-mail-address to; - { inherit from; - to = if typeOf to == "list" - then concatMapStringsSep "," (getAttr "mail") to - else to.mail; }; + to = concatMapStringsSep "," (getAttr "mail") (toList to); + }; in mapAttrsToList format (with config.krebs.users; let spam-ml = [ lass @@ -154,6 +166,10 @@ let "makefu@retiolum" = makefu; "spam@retiolum" = spam-ml; "tv@retiolum" = tv; + "lass@r" = lass; + "makefu@r" = makefu; + "spam@r" = spam-ml; + "tv@r" = tv; }); services.openssh.hostKeys = diff --git a/krebs/3modules/exim-retiolum.nix b/krebs/3modules/exim-retiolum.nix index 1722eef1f..6e6928f89 100644 --- a/krebs/3modules/exim-retiolum.nix +++ b/krebs/3modules/exim-retiolum.nix @@ -11,6 +11,24 @@ let api = { enable = mkEnableOption "krebs.exim-retiolum"; + local_domains = mkOption { + type = with types; listOf hostname; + default = ["localhost"] ++ config.krebs.build.host.nets.retiolum.aliases; + }; + primary_hostname = mkOption { + type = types.str; + default = let x = "${config.krebs.build.host.name}.r"; in + assert elem x config.krebs.build.host.nets.retiolum.aliases; + x; + }; + relay_to_domains = mkOption { + # TODO hostname with wildcards + type = with types; listOf str; + default = [ + "*.r" + "*.retiolum" + ]; + }; }; imp = { @@ -21,9 +39,9 @@ let # TODO modular configuration assert config.krebs.retiolum.enable; '' - primary_hostname = ${retiolumHostname} - domainlist local_domains = @ : localhost - domainlist relay_to_domains = *.retiolum + primary_hostname = ${cfg.primary_hostname} + domainlist local_domains = ${concatStringsSep ":" cfg.local_domains} + domainlist relay_to_domains = ${concatStringsSep ":" cfg.relay_to_domains} hostlist relay_from_hosts = <; 127.0.0.1 ; ::1 acl_smtp_rcpt = acl_check_rcpt @@ -85,7 +103,7 @@ let retiolum: driver = manualroute - domains = ! ${retiolumHostname} : *.retiolum + domains = ! +local_domains : +relay_to_domains transport = remote_smtp route_list = ^.* $0 byname no_more @@ -125,8 +143,8 @@ let # mode = 0660 begin retry - *.retiolum * F,42d,1m - * * F,2h,15m; G,16h,1h,1.5; F,4d,6h + ${concatMapStringsSep "\n" (k: "${k} * F,42d,1m") cfg.relay_to_domains} + * * F,2h,15m; G,16h,1h,1.5; F,4d,6h begin rewrite @@ -134,8 +152,4 @@ let ''; }; }; - - # TODO get the hostname from somewhere else. - retiolumHostname = "${config.networking.hostName}.retiolum"; -in -out +in out diff --git a/krebs/3modules/exim-smarthost.nix b/krebs/3modules/exim-smarthost.nix index 267ee2900..c976e89de 100644 --- a/krebs/3modules/exim-smarthost.nix +++ b/krebs/3modules/exim-smarthost.nix @@ -25,14 +25,31 @@ let })); }; + local_domains = mkOption { + type = with types; listOf hostname; + default = ["localhost"] ++ config.krebs.build.host.nets.retiolum.aliases; + }; + relay_from_hosts = mkOption { type = with types; listOf str; default = []; + apply = xs: ["127.0.0.1" "::1"] ++ xs; + }; + + relay_to_domains = mkOption { + # TODO hostname with wildcards + type = with types; listOf str; + default = [ + "*.r" + "*.retiolum" + ]; }; primary_hostname = mkOption { type = types.str; - default = "${config.networking.hostName}.retiolum"; + default = let x = "${config.krebs.build.host.name}.r"; in + assert elem x config.krebs.build.host.nets.retiolum.aliases; + x; }; sender_domains = mkOption { @@ -63,19 +80,11 @@ let # HOST_REDIR contains the real destinations for "local_domains". #HOST_REDIR = /etc/exim4/host_redirect - # Domains not listed in local_domains need to be deliverable remotely. # XXX We abuse local_domains to mean "domains, we're the gateway for". - domainlist local_domains = @ : localhost - domainlist relay_to_domains = - hostlist relay_from_hosts = <;${concatStringsSep ";" ( - [ - "127.0.0.1" - "::1" - ] - ++ - cfg.relay_from_hosts - )} + domainlist local_domains = ${concatStringsSep ":" cfg.local_domains} + domainlist relay_to_domains = ${concatStringsSep ":" cfg.relay_to_domains} + hostlist relay_from_hosts = <;${concatStringsSep ";" cfg.relay_from_hosts} acl_smtp_rcpt = acl_check_rcpt acl_smtp_data = acl_check_data @@ -144,7 +153,7 @@ let retiolum: debug_print = "R: retiolum for $local_part@$domain" driver = manualroute - domains = ! ${cfg.primary_hostname} : *.retiolum + domains = ! +local_domains : +relay_to_domains transport = retiolum_smtp route_list = ^.* $0 byname no_more @@ -197,8 +206,11 @@ let return_path_add begin retry - *.retiolum * F,42d,1m - * * F,2h,15m; G,16h,1h,1.5; F,4d,6h + ${concatMapStringsSep "\n" (k: "${k} * F,42d,1m") cfg.relay_to_domains} + ${concatMapStringsSep "\n" (k: "${k} * F,42d,1m") + # TODO don't include relay_to_domains< |