diff options
31 files changed, 611 insertions, 394 deletions
@@ -1,7 +1,8 @@ # # usage: -# make system=foo -# make systems='foo bar' +# make infest system=foo [target=bar] +# make [deploy] system=foo [target=bar] +# make [deploy] systems='foo bar' # make eval get=tv.wu.config.time.timeZone [filter=json] # @@ -11,6 +12,7 @@ ifdef systems $(systems): @ + unset target parallel \ --line-buffer \ -j0 \ @@ -20,7 +22,7 @@ $(systems): else ifdef system .PHONY: deploy infest deploy infest:;@ - export get=$$LOGNAME.${system}.config.krebs.build.scripts.$@ + export get=krebs.$@ export filter=json make -s eval | sh @@ -39,8 +41,11 @@ endif --eval \ -A "$$get" \ '<stockholm>' \ - --argstr user-name "$$LOGNAME" \ - --argstr host-name "$$HOSTNAME" \ + --argstr current-date "$$(date -Is)" \ + --argstr current-host-name "$$HOSTNAME" \ + --argstr current-user-name "$$LOGNAME" \ + $${system+--argstr system "$$system"} \ + $${target+--argstr target "$$target"} \ | filter else $(error unbound variable: system[s]) diff --git a/default.nix b/default.nix index 1c3341ba7..c6a635c29 100644 --- a/default.nix +++ b/default.nix @@ -1,36 +1,34 @@ -{ user-name, host-name }: +{ current-date +, current-host-name +, current-user-name +}: let lib = import <nixpkgs/lib>; krebs-modules-path = ./krebs/3modules; krebs-pkgs-path = ./krebs/5pkgs; - user-modules-path = ./. + "/${user-name}/3modules"; - user-pkgs-path = ./. + "/${user-name}/5pkgs"; + user-modules-path = ./. + "/${current-user-name}/3modules"; + user-pkgs-path = ./. + "/${current-user-name}/5pkgs"; out = - (lib.mapAttrs (k: v: mk-namespace (./. + "/${k}")) - (lib.filterAttrs - (k: v: !lib.hasPrefix "." k && v == "directory" && - builtins.pathExists (./. + "/${k}/1systems")) - (builtins.readDir ./.))); + lib.mapAttrs (_: builtins.getAttr "main") + (lib.filterAttrs (_: builtins.hasAttr "main") + (lib.mapAttrs + (k: v: + if lib.hasPrefix "." k || v != "directory" then + {} + else if builtins.pathExists (./. + "/${k}/default.nix") then + { main = import (./. + "/${k}"); } + else if builtins.pathExists (./. + "/${k}/1systems") then + { main = mk-namespace (./. + "/${k}"); } + else + {}) + (builtins.readDir ./.))); eval = path: import <nixpkgs/nixos/lib/eval-config.nix> { system = builtins.currentSystem; modules = [ - ({ config, ... }: - with import ./krebs/4lib { inherit lib; }; - { - options.krebs.exec.host = mkOption { - type = types.host; - default = config.krebs.hosts.${host-name}; - }; - options.krebs.exec.user = mkOption { - type = types.user; - default = config.krebs.users.${user-name}; - }; - } - ) path krebs-modules-path user-modules-path diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix new file mode 100644 index 000000000..57495ea69 --- /dev/null +++ b/krebs/3modules/build.nix @@ -0,0 +1,72 @@ +{ config, lib, ... }: + +with import ../4lib { inherit lib; }; + +let + target = config.krebs.build // { user.name = "root"; }; + + out = { + # TODO deprecate krebs.build.host + options.krebs.build.host = mkOption { + type = types.host; + }; + + # TODO make krebs.build.profile shell safe + options.krebs.build.profile = mkOption { + type = types.str; + default = "/nix/var/nix/profiles/system"; + }; + + # TODO make krebs.build.target.host :: host + options.krebs.build.target = mkOption { + type = with types; nullOr str; + default = null; + }; + + # TODO deprecate krebs.build.user + options.krebs.build.user = mkOption { + type = types.user; + }; + + options.krebs.build.source.dir = mkOption { + type = types.attrsOf (types.submodule ({ config, ... }: { + options = { + host = mkOption { + type = types.host; + }; + path = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + url = mkOption { + type = types.str; + default = "file://${config.host.name}${config.path}"; + }; + }; + })); + default = {}; + }; + + options.krebs.build.source.git = mkOption { + type = with types; attrsOf (submodule ({ config, ... }: { + options = { + url = mkOption { + type = types.str; # TODO must be shell safe + }; + rev = mkOption { + type = types.str; + }; + target-path = mkOption { + type = types.str; + default = "/root/${config._module.args.name}"; + }; + }; + })); + default = {}; + }; + }; + +in out diff --git a/krebs/3modules/build/default.nix b/krebs/3modules/build/default.nix deleted file mode 100644 index 6097a7b5d..000000000 --- a/krebs/3modules/build/default.nix +++ /dev/null @@ -1,309 +0,0 @@ -{ config, lib, ... }: - -with import ../../4lib { inherit lib; }; - -let - target = config.krebs.build // { user.name = "root"; }; - - out = { - # TODO deprecate krebs.build.host - options.krebs.build.host = mkOption { - type = types.host; - }; - - # TODO make krebs.build.profile shell safe - options.krebs.build.profile = mkOption { - type = types.str; - default = "/nix/var/nix/profiles/system"; - }; - - # TODO make krebs.build.target.host :: host - options.krebs.build.target = mkOption { - type = with types; nullOr str; - default = null; - }; - - # TODO deprecate krebs.build.user - options.krebs.build.user = mkOption { - type = types.user; - }; - - options.krebs.build.scripts.init = lib.mkOption { - type = lib.types.str; - default = - let - inherit (config.krebs.build) host; - in - '' - #! /bin/sh - set -efu - - hostname=${host.name} - secrets_dir=${config.krebs.build.source.dir.secrets.path} - key_type=ed25519 - key_file=$secrets_dir/ssh.id_$key_type - key_comment=$hostname - - if test -e "$key_file"; then - echo "Warning: privkey already exists: $key_file" >&2 - else - ssh-keygen \ - -C "$key_comment" \ - -t "$key_type" \ - -f "$key_file" \ - -N "" - rm "$key_file.pub" - fi - - pubkey=$(ssh-keygen -y -f "$key_file") - - cat<<EOF - # put following into config.krebs.hosts.$hostname: - ssh.pubkey = $(echo $pubkey | jq -R .); - EOF - ''; - }; - - options.krebs.build.scripts.deploy = lib.mkOption { - type = lib.types.str; - default = '' - set -efu - (${config.krebs.build.scripts._source}) - ${ssh-target '' - ${config.krebs.build.scripts._nix-env} - ${config.krebs.build.profile}/bin/switch-to-configuration switch - ''} - echo OK - ''; - }; - - options.krebs.build.scripts.infest = lib.mkOption { - type = lib.types.str; - default = '' - set -efu - - export RSYNC_RSH; RSYNC_RSH="$(type -p ssh) \ - -o 'HostName ${target.host.infest.addr}' \ - -o 'Port ${toString target.host.infest.port}' \ - " - ssh() { - eval "$RSYNC_RSH \"\$@\"" - } - - ${ssh-target '' - ${readFile ./infest/prepare.sh} - ${readFile ./infest/install-nix.sh} - ''} - - (${config.krebs.build.scripts._source}) - - ${ssh-target '' - export PATH; PATH=/root/.nix-profile/bin:$PATH - - src=$(type -p nixos-install) - cat_src() { - sed < "$src" "$( - { sed < "$src" -n ' - /^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/= - /^nixpkgs=/= - /^NIX_PATH=/,/^$/{/./=} - - # Disable: Copy the NixOS/Nixpkgs sources to the target as - # the initial contents of the NixOS channel. - /^srcs=/,/^ln -sfn /= - ' - } | sed 's:$:s/^/#krebs#/:' - )" - } - - # Location to insert config.krebs.build.scripts._nix-env - i=$(sed -n '/^echo "building the system configuration/=' "$src") - - { - cat_src | sed -n "1,$i{p}" - cat ${doc config.krebs.build.scripts._nix-env} - cat_src | sed -n "$i,\''${$i!p}" - } > nixos-install - chmod +x nixos-install - - # Wrap inserted config.krebs.build.scripts._nix-env into chroot. - nix_env=$(cat_src | sed -n ' - s:.*\(/nix/store/[a-z0-9]*-nix-[0-9.]\+/bin/nix-env\).*:\1:p;T;q - ') - echo nix-env is $nix_env - sed -i ' - s:^nix-env:chroot $mountPoint '"$nix_env"': - ' nixos-install - - ./nixos-install - - ${readFile ./infest/finalize.sh} - ''} - ''; - }; - - options.krebs.build.scripts._nix-env = lib.mkOption { - type = lib.types.str; - default = '' - set -efu - NIX_PATH=${config.krebs.build.source.NIX_PATH} \ - nix-env \ - -f '<stockholm>' \ - -Q \ - --argstr user-name ${config.krebs.exec.user.name} \ - --argstr host-name ${target.host.name} \ - --profile ${config.krebs.build.profile} \ - --set \ - -A ${lib.escapeShellArg (lib.concatStringsSep "." [ - config.krebs.build.user.name - config.krebs.build.host.name - "system" - ])} - ''; - }; - - options.krebs.build.scripts._source = lib.mkOption { - type = lib.types.str; - default = '' - set -efu - ${ - lib.concatStringsSep "\n" - (lib.mapAttrsToList - (name: { scripts, url, ... }: "(${scripts._source})") - (config.krebs.build.source.dir // - config.krebs.build.source.git)) - } - ''; - }; - - options.krebs.build.source.NIX_PATH = mkOption { - type = types.str; - default = - lib.concatStringsSep ":" - (lib.mapAttrsToList (name: _: "${name}=/root/${name}") - (config.krebs.build.source.dir // - config.krebs.build.source.git)); - }; - - options.krebs.build.source.dir = mkOption { - type = - let - exec = config.krebs.exec; - in - types.attrsOf (types.submodule ({ config, ... }: - let - url = "file://${config.host.name}${config.path}"; - - can-link = config.host.name == target.host.name; - can-push = config.host.name == exec.host.name; - - push-method = '' - rsync \ - --exclude .git \ - --exclude .graveyard \ - --exclude old \ - --exclude tmp \ - --rsync-path='mkdir -p ${config.target-path} && rsync' \ - --delete-excluded \ - -vrLptgoD \ - ${config.path}/ \ - ${target.user.name}@${target.host.name}:${config.target-path} - ''; - in - { - options = { - host = mkOption { - type = types.host; - description = '' - define the host where the directory is stored on. - XXX: currently it is just used to check if rsync is working, - becomes part of url - ''; - }; - path = mkOption { - type = types.str; - }; - scripts._source = mkOption { - type = types.str; - default = - #if can-link then link-method else - if can-push then push-method else - throw "cannot source ${url}"; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; - url = mkOption { - type = types.str; - default = "file://${config.host.name}${config.path}"; - }; - }; - } - )); - default = {}; - }; - - options.krebs.build.source.git = mkOption { - type = - let - target = config.krebs.build // { user.name = "root"; }; - in - with types; attrsOf (submodule ({ config, ... }: - { - options = { - url = mkOption { - type = types.str; # TODO must be shell safe - }; - rev = mkOption { - type = types.str; - }; - scripts._source = mkOption { - type = types.str; - default = ssh-target '' - mkdir -p ${config.target-path} - cd ${config.target-path} - if ! test -e .git; then - git init - fi - if ! cur_url=$(git config remote.origin.url 2>/dev/null); then - git remote add origin ${config.url} - elif test "$cur_url" != ${config.url}; then - git remote set-url origin ${config.url} - fi - if test "$(git rev-parse --verify HEAD 2>/dev/null)" != ${config.rev}; then - git fetch origin - git checkout ${config.rev} -- . - git checkout -q ${config.rev} - git submodule init - git submodule update - fi - git clean -dxf - ''; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; - }; - } - )); - default = {}; - }; - }; - - doc = s: - let b = "EOF${hashString "sha256" s}"; in - '' - <<\${b} - ${s} - ${b} - ''; - - ssh-target = script: - "ssh root@${target.host.name} -T ${doc '' - set -efu - ${script} - ''}"; - -in out diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index a1ce2ad6c..622432dab 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -6,7 +6,7 @@ let out = { imports = [ - ./build + ./build.nix ./exim-retiolum.nix ./exim-smarthost.nix ./github-hosts-sync.nix @@ -143,7 +143,7 @@ let dc = "lass"; #dc = "cac"; nets = rec { internet = { - addrs4 = ["162.248.8.63"]; + addrs4 = ["104.233.84.57"]; aliases = [ "echelon.internet" ]; @@ -158,12 +158,42 @@ let ]; tinc.pubkey = '' -----BEGIN RSA PUBLIC KEY----- - MIIBCgKCAQEA92ybhDahtGybpAkUNlG5Elxw05MVY4Pg7yK0dQugB4nVq+pnmi78 - DOMeIciecMHmJM8n9UlUU0eWZVCgHeVd23d6J0hTHCv24p24uHEGGy7XlO/dPJ6A - IjROYU0l8c03pipdJ3cDBx6riArSglwmZJ7xH/Iw0BUhRZrPqbtijY7EcG2wc+8K - N9N9mBofVMl4EcBiDR/eecK+ro8OkeOmYPtYgFJLvxTYXiPIhOxMAlkOY2fpin/t - cgFLUFuN4ag751XjjcNpVovVq95vdg+VhKrrNVWZjJt03owW81BzoryY6CD2kIPq - UxK89zEdeYOUT7AxaT/5V5v41IvGFZxCzwIDAQAB + MIIBCgKCAQEAuscWOYdHu0bpWacvwTNd6bcmrAQ0YFxJWHZF8kPZr+bMKIhnXLkJ + oJheENIM6CA9lQQQFUxh2P2pxZavW5rgVlJxIKeiB+MB4v6ZO60LmZgpCsWGD/dX + MipM2tLtQxYhvLJIJxEBWn3rxIgeEnCtZsH1KLWyLczb+QpvTjMJ4TNh1nEBPE/f + 4LUH1JHaGhcaHl2dLemR9wnnDIjmSj0ENJp2al+hWnIggcA/Zp0e4b86Oqbbs5wA + n++n5j971cTrBdA89nJDYOEtepisglScVRbgLqJG81lDA+n24RWFynn+U3oD/L8p + do+kxlwZUEDRbPU4AO5L+UeIbimsuIfXiQIDAQAB + -----END RSA PUBLIC KEY----- + ''; + }; + }; + }; + fastpoke = { + dc = "lass"; #dc = "cac"; + nets = rec { + internet = { + addrs4 = ["193.22.164.36"]; + aliases = [ + "fastpoke.internet" + ]; + }; + retiolum = { + via = internet; + addrs4 = ["10.243.253.152"]; + addrs6 = ["42:422a:194f:ff3b:e196:2f82:5cf5:bc00"]; + aliases = [ + "fastpoke.retiolum" + "cgit.fastpoke.retiolum" + ]; + tinc.pubkey = '' + -----BEGIN RSA PUBLIC KEY----- + MIIBCgKCAQEAs4p5xsQYx06v+OkUbc09K6voFAbkvO66QdFoM71E10XyCeLP6iuq + DaIOFN4GrPR36pgyjqtJ+62G9uR+WsB/y14eio1p1ivDWgcpt5soOZAH5zVRRD9O + FBDlgVNwIJ6stMHy6OenEKWsfEiZRN3XstnqAqyykzjddglth1tJntn6kbZehzNQ + ezfIyN4XgaX2fhSu+UnAyLcV8wWnF9cMABjz7eKcSmRJgtG4ZiuDkbgiiEew7+pB + EPqOVQ80lJvzQKgO4PmVoAjD9A+AHnmLJNPDQQi8nIVilGCT60IX+XT1rt85Zpdy + rEaeriw/qsVJnberAhDAdQYYuM1ai2H5swIDAQAB -----END RSA PUBLIC KEY----- ''; }; @@ -568,10 +598,9 @@ let mkdir = rec { cores = 1; dc = "tv"; #dc = "cac"; - infest.addr = head nets.internet.addrs4; nets = rec { internet = { - addrs4 = ["104.233.84.173"]; + addrs4 = ["104.233.84.215"]; aliases = [ "mkdir.internet" ]; @@ -655,7 +684,6 @@ let rmdir = rec { cores = 1; dc = "tv"; #dc = "cac"; - infest.addr = head nets.internet.addrs4; nets = rec { internet = { addrs4 = ["104.233.84.70"]; diff --git a/krebs/3modules/build/infest/finalize.sh b/krebs/4lib/infest/finalize.sh index ced5a4d4d..ced5a4d4d 100644 --- a/krebs/3modules/build/infest/finalize.sh +++ b/krebs/4lib/infest/finalize.sh diff --git a/krebs/3modules/build/infest/install-nix.sh b/krebs/4lib/infest/install-nix.sh index 88c8c3e1e..88c8c3e1e 100644 --- a/krebs/3modules/build/infest/install-nix.sh +++ b/krebs/4lib/infest/install-nix.sh diff --git a/krebs/3modules/build/infest/prepare.sh b/krebs/4lib/infest/prepare.sh index 07c00c3a5..07c00c3a5 100644 --- a/krebs/3modules/build/infest/prepare.sh +++ b/krebs/4lib/infest/prepare.sh diff --git a/krebs/4lib/shell.nix b/krebs/4lib/shell.nix index 2a6da5c16..5910adacc 100644 --- a/krebs/4lib/shell.nix +++ b/krebs/4lib/shell.nix @@ -6,7 +6,7 @@ with lib; rec { escape = let - isSafeChar = c: match "[-./0-9_a-zA-Z]" c != null; + isSafeChar = c: match "[-+./0-9:=A-Z_a-z]" c != null; in stringAsChars (c: if isSafeChar c then c diff --git a/krebs/4lib/types.nix b/krebs/4lib/types.nix index 0aa594fb1..039f803ef 100644 --- a/krebs/4lib/types.nix +++ b/krebs/4lib/types.nix @@ -31,10 +31,13 @@ types // rec { infest = { addr = mkOption { type = str; + apply = trace "Obsolete option `krebs.hosts.${config.name}.infest.addr' is used. It was replaced by the `target' argument to `make` or `get`. See Makefile for more information."; }; port = mkOption { type = int; default = 22; + # TODO replacement: allow target with port, SSH-style: [lol]:666 + apply = trace "Obsolete option `krebs.hosts.${config.name}.infest.port' is used. It's gone without replacement."; }; }; diff --git a/krebs/5pkgs/get/default.nix b/krebs/5pkgs/get/default.nix index e2591db73..20bbfd014 100644 --- a/krebs/5pkgs/get/default.nix +++ b/krebs/5pkgs/get/default.nix @@ -1,12 +1,12 @@ { coreutils, gnugrep, gnused, fetchgit, jq, nix, stdenv, ... }: stdenv.mkDerivation { - name = "get-1.1.1"; + name = "get-1.3.0"; src = fetchgit { url = http://cgit.cd.retiolum/get; - rev = "e64826a4f5f74cbaa895e538b97d0e523e9709f9"; - sha256 = "4d1aa07bba52f697cf7aa7ad1b02b9ff41598dfea83c578e77b8d81e3e8830d2"; + rev = "fbe8f8d12ede9762fceb15b9944b69a4ee6331eb"; + sha256 = "bcdf036f8b5d1467285d0998aeac7e48280adfb9e1278f9f424c9c8b5e6ed8fa"; }; phases = [ diff --git a/krebs/Zhosts/cloudkrebs b/krebs/Zhosts/cloudkrebs index ed46a36bd..3886371ff 100644 --- a/krebs/Zhosts/cloudkrebs +++ b/krebs/Zhosts/cloudkrebs @@ -1,4 +1,4 @@ -Address = 167.88.34.190 +Address = 104.167.113.104 Subnet = 10.243.206.102 Subnet = 42:941e:2816:35f4:5c5e:206b:3f0b:f762 diff --git a/krebs/Zhosts/echelon b/krebs/Zhosts/echelon index 9d1c324fd..de4366875 100644 --- a/krebs/Zhosts/echelon +++ b/krebs/Zhosts/echelon @@ -1,12 +1,12 @@ -Address = 168.235.156.81 +Address = 104.233.84.57 Subnet = 10.243.206.103 Subnet = 42:941e:2816:35f4:5c5e:206b:3f0b:f763 -----BEGIN RSA PUBLIC KEY----- -MIIBCgKCAQEA92ybhDahtGybpAkUNlG5Elxw05MVY4Pg7yK0dQugB4nVq+pnmi78 -DOMeIciecMHmJM8n9UlUU0eWZVCgHeVd23d6J0hTHCv24p24uHEGGy7XlO/dPJ6A -IjROYU0l8c03pipdJ3cDBx6riArSglwmZJ7xH/Iw0BUhRZrPqbtijY7EcG2wc+8K -N9N9mBofVMl4EcBiDR/eecK+ro8OkeOmYPtYgFJLvxTYXiPIhOxMAlkOY2fpin/t -cgFLUFuN4ag751XjjcNpVovVq95vdg+VhKrrNVWZjJt03owW81BzoryY6CD2kIPq -UxK89zEdeYOUT7AxaT/5V5v41IvGFZxCzwIDAQAB +MIIBCgKCAQEAuscWOYdHu0bpWacvwTNd6bcmrAQ0YFxJWHZF8kPZr+bMKIhnXLkJ +oJheENIM6CA9lQQQFUxh2P2pxZavW5rgVlJxIKeiB+MB4v6ZO60LmZgpCsWGD/dX +MipM2tLtQxYhvLJIJxEBWn3rxIgeEnCtZsH1KLWyLczb+QpvTjMJ4TNh1nEBPE/f +4LUH1JHaGhcaHl2dLemR9wnnDIjmSj0ENJp2al+hWnIggcA/Zp0e4b86Oqbbs5wA +n++n5j971cTrBdA89nJDYOEtepisglScVRbgLqJG81lDA+n24RWFynn+U3oD/L8p +do+kxlwZUEDRbPU4AO5L+UeIbimsuIfXiQIDAQAB -----END RSA PUBLIC KEY----- diff --git a/krebs/default.nix b/krebs/default.nix new file mode 100644 index 000000000..0ec4c6079 --- /dev/null +++ b/krebs/default.nix @@ -0,0 +1,263 @@ +{ current-date +, current-host-name +, current-user-name +}@current: rec { + + deploy = + { system ? current-host-name + , target ? system + }@args: let + config = lib.get-config system; + in '' + #! /bin/sh + # ${current-date} ${current-user-name}@${current-host-name} + # krebs.deploy + set -efu + (${lib.populate args}) + ${lib.rootssh target '' + ${lib.install args} + ${config.krebs.build.profile}/bin/switch-to-configuration switch + ''} + echo OK + ''; + + infest = + { system ? current-host-name + , target ? system + }@args: let + in '' + #! /bin/sh + # ${current-date} ${current-user-name}@${current-host-name} + # krebs.infest + set -efu + + # XXX type -p is non-standard + #export RSYNC_RSH; RSYNC_RSH="$(type -p ssh) \ + # -o 'HostName $ {target.host.infest.addr}' \ + # -o 'Port $ {toString target.host.infest.port}' \ + #" + #ssh() { + # eval "$RSYNC_RSH \"\$@\"" + #} + + ${lib.rootssh target '' + ${builtins.readFile ./4lib/infest/prepare.sh} + ${builtins.readFile ./4lib/infest/install-nix.sh} + ''} + + (${lib.populate args}) + + ${lib.rootssh target '' + export PATH; PATH=/root/.nix-profile/bin:$PATH + + src=$(type -p nixos-install) + cat_src() { + sed < "$src" "$( + { sed < "$src" -n ' + /^if ! test -e "\$mountPoint\/\$NIXOS_CONFIG/,/^fi$/= + /^nixpkgs=/= + /^NIX_PATH=/,/^$/{/./=} + + # Disable: Copy the NixOS/Nixpkgs sources to the target as + # the initial contents of the NixOS channel. + /^srcs=/,/^ln -sfn /= + ' + } | sed 's:$:s/^/#krebs#/:' + )" + } + + # Location to insert lib.install + i=$(sed -n '/^echo "building the system configuration/=' "$src") + + { |