sync-containers3: lass -> krebs

13 files changed, 11 insertions, 359 deletions

diff --git a/lass/1systems/green/config.nix b/lass/1systems/green/config.nix
index cd38c3585..077f7b3fa 100644
--- a/lass/1systems/green/config.nix
+++ b/lass/1systems/green/config.nix
@@ -27,7 +27,7 @@ with import <stockholm/lib>;
 
   krebs.build.host = config.krebs.hosts.green;
 
-  lass.sync-containers3.inContainer = {
+  krebs.sync-containers3.inContainer = {
     enable = true;
     pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFlUMf943qEQG64ob81p6dgoHq4jUjq7tSvmSdEOEU2y";
   };
diff --git a/lass/1systems/orange/config.nix b/lass/1systems/orange/config.nix
index 3c13ebe85..3bc20878e 100644
--- a/lass/1systems/orange/config.nix
+++ b/lass/1systems/orange/config.nix
@@ -14,7 +14,7 @@ with import <stockholm/lib>;
     defaults.email = "acme@lassul.us";
   };
 
-  lass.sync-containers3.inContainer = {
+  krebs.sync-containers3.inContainer = {
     enable = true;
     pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFQWzKuXrwQopBc1mzb2VpljmwAs7Y8bRl9a8hBXLC+l";
   };
diff --git a/lass/1systems/radio/config.nix b/lass/1systems/radio/config.nix
index 2fd23a448..5e34335d3 100644
--- a/lass/1systems/radio/config.nix
+++ b/lass/1systems/radio/config.nix
@@ -17,7 +17,7 @@ with import <stockholm/lib>;
     defaults.email = "acme@lassul.us";
   };
 
-  lass.sync-containers3.inContainer = {
+  krebs.sync-containers3.inContainer = {
     enable = true;
     pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOvPKdbVwMEFCDMyNAzR8NdVjTbQL2G+03Xomxn6KKFt";
   };
diff --git a/lass/1systems/ubik/config.nix b/lass/1systems/ubik/config.nix
index 1d1d32f3f..1d836d4ec 100644
--- a/lass/1systems/ubik/config.nix
+++ b/lass/1systems/ubik/config.nix
@@ -9,7 +9,7 @@ with import <stockholm/lib>;
 
   krebs.build.host = config.krebs.hosts.ubik;
 
-  lass.sync-containers3.inContainer = {
+  krebs.sync-containers3.inContainer = {
     enable = true;
     pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPBFGMjH0+Dco6DVFZbByENMci8CFTLXCL7j53yctPnM";
   };
diff --git a/lass/1systems/yellow/config.nix b/lass/1systems/yellow/config.nix
index 73d7f3780..ff8189e24 100644
--- a/lass/1systems/yellow/config.nix
+++ b/lass/1systems/yellow/config.nix
@@ -9,7 +9,7 @@ in {
 
   krebs.build.host = config.krebs.hosts.yellow;
 
-  lass.sync-containers3.inContainer = {
+  krebs.sync-containers3.inContainer = {
     enable = true;
     pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN737BAP36KiZO97mPKTIUGJUcr97ps8zjfFag6cUiYL";
   };
diff --git a/lass/2configs/green-host.nix b/lass/2configs/green-host.nix
index 1e41e8e02..66088a562 100644
--- a/lass/2configs/green-host.nix
+++ b/lass/2configs/green-host.nix
@@ -1,10 +1,6 @@
 { config, pkgs, ... }:
 {
-  imports = [
-    <stockholm/lass/2configs/container-networking.nix>
-  ];
-
-  lass.sync-containers3.containers.green = {
+  krebs.sync-containers3.containers.green = {
     sshKey = "${toString <secrets>}/green.sync.key";
   };
 }
diff --git a/lass/2configs/orange-host.nix b/lass/2configs/orange-host.nix
index 3fbf417a7..e4bfcff89 100644
--- a/lass/2configs/orange-host.nix
+++ b/lass/2configs/orange-host.nix
@@ -1,6 +1,6 @@
 { config, pkgs, ... }:
 {
-  lass.sync-containers3.containers.orange = {
+  krebs.sync-containers3.containers.orange = {
     sshKey = "${toString <secrets>}/orange.sync.key";
   };
   services.nginx.virtualHosts."lassul.us" = {
diff --git a/lass/2configs/radio/container-host.nix b/lass/2configs/radio/container-host.nix
index e32095ffa..de0ea9afe 100644
--- a/lass/2configs/radio/container-host.nix
+++ b/lass/2configs/radio/container-host.nix
@@ -1,6 +1,6 @@
 { config, pkgs, ... }:
 {
-  lass.sync-containers3.containers.radio = {
+  krebs.sync-containers3.containers.radio = {
     sshKey = "${toString <secrets>}/radio.sync.key";
   };
   containers.radio = {
diff --git a/lass/2configs/red-host.nix b/lass/2configs/red-host.nix
index cbd9c097e..171191dac 100644
--- a/lass/2configs/red-host.nix
+++ b/lass/2configs/red-host.nix
@@ -8,7 +8,7 @@ in
   ];
 
 
-  lass.sync-containers3.containers.red = {
+  krebs.sync-containers3.containers.red = {
     sshKey = "${toString <secrets>}/containers/red/sync.key";
     ephemeral = true;
   };
diff --git a/lass/2configs/ubik-host.nix b/lass/2configs/ubik-host.nix
index 874d4ecb8..a4ad5e55e 100644
--- a/lass/2configs/ubik-host.nix
+++ b/lass/2configs/ubik-host.nix
@@ -1,6 +1,6 @@
 { config, pkgs, ... }:
 {
-  lass.sync-containers3.containers.ubik = {
+  krebs.sync-containers3.containers.ubik = {
     sshKey = "${toString <secrets>}/ubik.sync.key";
   };
   containers.ubik.bindMounts."/var/lib" = {
diff --git a/lass/2configs/yellow-host.nix b/lass/2configs/yellow-host.nix
index d07c222c6..36027cb5d 100644
--- a/lass/2configs/yellow-host.nix
+++ b/lass/2configs/yellow-host.nix
@@ -1,6 +1,6 @@
 { config, pkgs, ... }:
 {
-  lass.sync-containers3.containers.yellow = {
+  krebs.sync-containers3.containers.yellow = {
     sshKey = "${toString <secrets>}/yellow.sync.key";
   };
   containers.yellow.bindMounts."/var/lib" = {
diff --git a/lass/3modules/default.nix b/lass/3modules/default.nix
index 42efa8cd6..3a0b1306c 100644
--- a/lass/3modules/default.nix
+++ b/lass/3modules/default.nix
@@ -15,6 +15,5 @@ _:
     ./xjail.nix
     ./autowifi.nix
     ./browsers.nix
-    ./sync-containers3.nix
   ];
 }
diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix
deleted file mode 100644
index 5188f270d..000000000
--- a/lass/3modules/sync-containers3.nix
+++ /dev/null
@@ -1,343 +0,0 @@
-{ config, lib, pkgs, ... }: let
-  cfg = config.lass.sync-containers3;
-  slib = pkgs.stockholm.lib;
-in {
-  options.lass.sync-containers3 = {
-    inContainer = {
-      enable = lib.mkEnableOption "container config for syncing";
-      pubkey = lib.mkOption {
-        type = lib.types.str; # TODO ssh key
-      };
-    };
-    containers = lib.mkOption {
-      default = {};
-      type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: {
-        options = {
-          name = lib.mkOption {
-            type = lib.types.str;
-            default = config._module.args.name;
-          };
-          sshKey = lib.mkOption {
-            type = slib.types.absolute-pathname;
-          };
-          luksKey = lib.mkOption {
-            type = slib.types.absolute-pathname;
-            default = config.sshKey;
-          };
-          ephemeral = lib.mkOption {
-            type = lib.types.bool;
-            default = false;
-          };
-          runContainer = lib.mkOption {
-            type = lib.types.bool;
-            default = true;
-          };
-        };
-      }));
-    };
-  };
-  config = lib.mkMerge [
-    (lib.mkIf (cfg.containers != {}) {
-
-      containers = lib.mapAttrs' (n: ctr: lib.nameValuePair ctr.name {
-        config = {
-          environment.systemPackages = [
-            pkgs.dhcpcd
-            pkgs.git
-            pkgs.jq
-          ];
-          networking.useDHCP = lib.mkForce true;
-          systemd.services.autoswitch = {
-            environment = {
-              NIX_REMOTE = "daemon";
-            };
-            wantedBy = [ "multi-user.target" ];
-            serviceConfig.ExecStart = pkgs.writers.writeDash "autoswitch" ''
-              set -efu
-              mkdir -p /var/state/var_src
-              ln -Tfrs /var/state/var_src /var/src
-              if test -e /var/src/nixos-config; then
-                /run/current-system/sw/bin/nixos-rebuild -I /var/src switch || :
-              fi
-            '';
-            unitConfig.X-StopOnRemoval = false;
-          };
-        };
-        autoStart = false;
-        enableTun = true;
-        ephemeral = ctr.ephemeral;
-        privateNetwork = true;
-        hostBridge = "ctr0";
-        bindMounts = {
-          "/var/lib/self/disk" = {
-            hostPath = "/var/lib/sync-containers3/${ctr.name}/disk";
-            isReadOnly = false;
-          };
-          "/var/state" = {
-            hostPath = "/var/lib/sync-containers3/${ctr.name}/state";
-            isReadOnly = false;
-          };
-        };
-      }) (lib.filterAttrs (_: ctr: ctr.runContainer) cfg.containers);
-
-      systemd.services = lib.foldr lib.recursiveUpdate {} (lib.flatten (map (ctr: [
-        { "${ctr.name}_syncer" = {
-          path = with pkgs; [
-            coreutils
-            consul
-            rsync
-            openssh
-            systemd
-          ];
-          startAt = "*:0/1";
-          serviceConfig = {
-            User = "${ctr.name}_container";
-            LoadCredential = [
-              "ssh_key:${ctr.sshKey}"
-            ];
-            ExecCondition = pkgs.writers.writeDash "${ctr.name}_checker" ''
-              set -efu
-              ! systemctl is-active --quiet container@${ctr.name}.service
-            '';
-            ExecStart = pkgs.writers.writeDash "${ctr.name}_syncer" ''
-              set -efux
-              consul lock sync_${ctr.name} ${pkgs.writers.writeDash "${ctr.name}-sync" ''
-                set -efux
-                if /run/wrappers/bin/ping -c 1 ${ctr.name}.r; then
-                  nice --adjustment=30 rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --timeout=30 container_sync@${ctr.name}.r:disk "$HOME"/disk
-                  rm -f "$HOME"/incomplete
-                fi
-              ''}
-            '';
-          };
-        }; }
-        { "${ctr.name}_watcher" = lib.mkIf ctr.runContainer {
-          path = with pkgs; [
-            coreutils
-            consul
-            cryptsetup
-            curl
-            mount
-            util-linux
-            jq
-            retry
-          ];
-          serviceConfig = {
-            ExecStart = pkgs.writers.writeDash "${ctr.name}_watcher" ''
-              set -efux
-              while sleep 5; do
-                # get the payload
-                # check if the host reacted recently
-                case $(curl -s -o /dev/null --retry 10 --retry-delay 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in
-                  404)
-                    echo 'got 404 from kv, should kill the container'
-                    break
-                    ;;
-                  500)
-                    echo 'got 500 from kv, will kill container'
-                    break
-                    ;;
-                  200)
-                    # echo 'got 200 from kv, will check payload'
-                    payload=$(consul kv get containers/${ctr.name}) || continue
-                    export payload
-                    if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then
-                      # echo 'we are the host, trying to reach container'
-                      if $(retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null); then
-                        # echo 'container is reachable, continueing'
-                        continue
-                      else
-                        # echo 'container seems dead, killing'
-                        break
-                      fi
-                    else
-                      echo 'we are not host, killing container'
-                      break
-                    fi
-                    ;;
-                  *)
-                    echo 'unknown state, continuing'
-                    continue
-                    ;;
-                esac
-              done
-              /run/current-system/sw/bin/nixos-container stop ${ctr.name} || :
-              umount /var/lib/sync-containers3/${ctr.name}/state || :
-              cryptsetup luksClose ${ctr.name} || :
-            '';
-          };
-        }; }
-        { "${ctr.name}_scheduler" = lib.mkIf ctr.runContainer {
-          wantedBy = [ "multi-user.target" ];
-          path = with pkgs; [
-            coreutils
-            consul
-            cryptsetup
-            mount
-            util-linux
-            curl
-            systemd
-            jq
-            retry
-            bc
-          ];
-          serviceConfig = {
-            Restart = "always";
-            RestartSec = "30s";
-            ExecStart = pkgs.writers.writeDash "${ctr.name}_scheduler" ''
-              set -efux
-              # get the payload
-              # check if the host reacted recently
-              case $(curl -s -o /dev/null --retry 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in
-                404)
-                  # echo 'got 404 from kv, will create container'
-                  ;;
-                500)
-                  # echo 'got 500 from kv, retrying again'
-                  exit 0
-                  ;;
-                200)
-                  # echo 'got 200 from kv, will check payload'
-                  export payload=$(consul kv get containers/${ctr.name})
-                  if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then
-                    echo 'we are the host, starting container'
-                  else
-                    # echo 'we are not host, checking timestamp'
-                    # if [ $(echo "$(date +%s) - $(jq -rn 'env.payload | fromjson.time') > 100" | bc) -eq 1 ]; then
-                    if [ "$(jq -rn 'env.payload | fromjson.time | now - tonumber > 100')" = 'true' ]; then
-                      echo 'last beacon is more than 100s ago, taking over'
-                    else
-                      # echo 'last beacon was recent. trying again'
-                      exit 0
-                    fi
-                  fi
-                  ;;
-                *)
-                  echo 'unknown state, bailing out'
-                  exit 0
-                  ;;
-              esac
-              consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null
-              consul lock -verbose -monitor-retry=100 -timeout 30s -name container_${ctr.name} container_${ctr.name} ${pkgs.writers.writeBash "${ctr.name}-start" ''
-                set -efu
-                cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} || :
-                mkdir -p /var/lib/sync-containers3/${ctr.name}/state
-                mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
-                /run/current-system/sw/bin/nixos-container start ${ctr.name}
-                # wait for system to become reachable for the first time
-                systemctl start ${ctr.name}_watcher.service
-                retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null
-                while systemctl is-active container@${ctr.name}.service >/devnull && /run/wrappers/bin/ping -q -c 3 ${ctr.name}.r >/dev/null; do
-                  consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null
-                  sleep 10
-                done
-              ''}
-            '';
-          };
-        }; }
-        { "container@${ctr.name}" = lib.mkIf ctr.runContainer {
-          serviceConfig = {
-            ExecStop = pkgs.writers.writeDash "remove_interface" ''
-              ${pkgs.iproute2}/bin/ip link del vb-${ctr.name}
-            '';
-          };
-        }; }
-      ]) (lib.attrValues cfg.containers)));
-
-      systemd.timers = lib.mapAttrs' (n: ctr: lib.nameValuePair "${ctr.name}_syncer" {
-        timerConfig = {
-          RandomizedDelaySec = 100;
-        };
-      }) cfg.containers;
-
-      users.groups = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" {
-      }) cfg.containers;
-      users.users = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" ({
-        group = "${ctr.name}_container";
-        isNormalUser = true;
-        uid = slib.genid_uint31 "container_${ctr.name}";
-        home = "/var/lib/sync-containers3/${ctr.name}";
-        createHome = true;
-        homeMode = "705";
-      })) cfg.containers;
-
-      environment.systemPackages = lib.mapAttrsToList (_: ctr: (pkgs.writers.writeDashBin "${ctr.name}_init" ''
-        set -efux
-        export PATH=${lib.makeBinPath [
-          pkgs.coreutils
-          pkgs.cryptsetup
-          pkgs.libxfs.bin
-        ]}:$PATH
-        truncate -s 5G /var/lib/sync-containers3/${ctr.name}/disk
-        cryptsetup luksFormat /var/lib/sync-containers3/${ctr.name}/disk ${ctr.luksKey}
-        cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name}
-        mkfs.xfs /dev/mapper/${ctr.name}
-        mkdir -p /var/lib/sync-containers3/${ctr.name}/state
-        mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
-        /run/current-system/sw/bin/nixos-container start ${ctr.name}
-        /run/current-system/sw/bin/nixos-container run ${ctr.name} -- ${pkgs.writeDash "init" ''
-          mkdir -p /var/state
-        ''}
-      '')) cfg.containers;
-    })
-    (lib.mkIf (cfg.containers != {}) {
-      # networking
-
-      # needed because otherwise we lose local dns
-      environment.etc."resolv.conf".source = lib.mkForce "/run/systemd/resolve/resolv.conf";
-
-      boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkForce 1;
-      systemd.network.networks.ctr0 = {
-        name = "ctr0";
-        address = [
-          "10.233.0.1/24"
-        ];
-        networkConfig = {
-          # IPForward = "yes";
-          # IPMasquerade = "both";
-          ConfigureWithoutCarrier = true;
-          DHCPServer = "yes";
-        };
-      };
-      systemd.network.netdevs.ctr0.netdevConfig = {
-        Kind = "bridge";
-        Name = "ctr0";
-      };
-      networking.networkmanager.unmanaged = [ "ctr0" ];
-      krebs.iptables.tables.filter.INPUT.rules = [
-        { predicate = "-i ctr0"; target = "ACCEPT"; }
-      ];
-      krebs.iptables.tables.filter.FORWARD.rules = [
-        { predicate = "-i ctr0"; target = "ACCEPT"; }
-        { predicate = "-o ctr0"; target = "ACCEPT"; }
-      ];
-      krebs.iptables.tables.nat.POSTROUTING.rules = [
-        { v6 = false; predicate = "-s 10.233.0.0/24"; target = "MASQUERADE"; }
-      ];
-    })
-    (lib.mkIf cfg.inContainer.enable {
-      users.groups.container_sync = {};
-      users.users.container_sync = {
-        group = "container_sync";
-        uid = slib.genid_uint31 "container_sync";
-        isNormalUser = true;
-        home = "/var/lib/self";
-        createHome = true;
-        openssh.authorizedKeys.keys = [
-          cfg.inContainer.pubkey
-        ];
-      };
-
-      networking.useHostResolvConf = false;
-      networking.useNetworkd = true;
-      systemd.network = {
-        enable = true;
-        networks.eth0 = {
-          matchConfig.Name = "eth0";
-          DHCP = "yes";
-          dhcpV4Config.UseDNS = true;
-        };
-      };
-    })
-  ];
-}