From 3aae16c8476602f39163b4a786f95ebd8806c71b Mon Sep 17 00:00:00 2001
From: lassulus <lassulus@lassul.us>
Date: Tue, 15 Nov 2022 15:27:17 +0100
Subject: l: init sync-containers3

---
 lass/3modules/sync-containers3.nix | 226 +++++++++++++++++++++++++++++++++++++
 1 file changed, 226 insertions(+)
 create mode 100644 lass/3modules/sync-containers3.nix

(limited to 'lass/3modules/sync-containers3.nix')

diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix
new file mode 100644
index 000000000..5b6fa37e1
--- /dev/null
+++ b/lass/3modules/sync-containers3.nix
@@ -0,0 +1,226 @@
+{ config, lib, pkgs, ... }: let
+  cfg = config.lass.sync-containers3;
+  slib = pkgs.stockholm.lib;
+in {
+  options.lass.sync-containers3 = {
+    inContainer = {
+      enable = lib.mkEnableOption "container config for syncing";
+      pubkey = lib.mkOption {
+        type = lib.types.str; # TODO ssh key
+      };
+    };
+    containers = lib.mkOption {
+      default = {};
+      type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: {
+        options = {
+          name = lib.mkOption {
+            type = lib.types.str;
+            default = config._module.args.name;
+          };
+          sshKey = lib.mkOption {
+            type = slib.types.absolute-pathname;
+          };
+          luksKey = lib.mkOption {
+            type = slib.types.absolute-pathname;
+            default = config.sshKey;
+          };
+          ephemeral = lib.mkOption {
+            type = lib.types.bool;
+            default = false;
+          };
+        };
+      }));
+    };
+  };
+  config = lib.mkMerge [
+    (lib.mkIf (cfg.containers != {}) {
+
+      containers = lib.mapAttrs' (n: ctr: lib.nameValuePair ctr.name {
+        config = {
+          environment.systemPackages = [
+            pkgs.dhcpcd
+            pkgs.git
+            pkgs.jq
+          ];
+          networking.useDHCP = lib.mkForce true;
+          systemd.services.autoswitch = {
+            environment = {
+              NIX_REMOTE = "daemon";
+            };
+            wantedBy = [ "multi-user.target" ];
+            serviceConfig.ExecStart = pkgs.writers.writeDash "autoswitch" ''
+              set -efu
+              ln -frs /var/state/var_src /var/src
+              if test -e /var/src/nixos-config; then
+                /run/current-system/sw/bin/nixos-rebuild -I /var/src switch || :
+              fi
+            '';
+            unitConfig.X-StopOnRemoval = false;
+          };
+        };
+        autoStart = false;
+        enableTun = true;
+        ephemeral = ctr.ephemeral;
+        privateNetwork = true;
+        hostBridge = "ctr0";
+        bindMounts = {
+          "/etc/resolv.conf".hostPath = "/etc/resolv.conf";
+          "/var/lib/self/disk" = {
+            hostPath = "/var/lib/sync-containers3/${ctr.name}/disk";
+            isReadOnly = false;
+          };
+          "/var/state" = {
+            hostPath = "/var/lib/sync-containers3/${ctr.name}/state";
+            isReadOnly = false;
+          };
+        };
+      }) cfg.containers;
+
+      systemd.services = lib.foldr lib.recursiveUpdate {} (lib.flatten (map (ctr: [
+        { "${ctr.name}_syncer" = {
+          path = with pkgs; [
+            coreutils
+            consul
+            rsync
+            openssh
+            systemd
+          ];
+          startAt = "*:0/1";
+          serviceConfig = {
+            User = "${ctr.name}_container";
+            LoadCredential = [
+              "ssh_key:${ctr.sshKey}"
+            ];
+            ExecCondition = pkgs.writers.writeDash "${ctr.name}_checker" ''
+              set -efu
+              ! systemctl is-active --quiet container@${ctr.name}.service
+            '';
+            ExecStart = pkgs.writers.writeDash "${ctr.name}_syncer" ''
+              set -efux
+              consul lock sync_${ctr.name} ${pkgs.writers.writeDash "${ctr.name}-sync" ''
+                set -efux
+                if ping -c 1 ${ctr.name}.r; then
+                  touch "$HOME"/incomplete
+                  rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --inplace container_sync@${ctr.name}.r:disk "$HOME"/disk
+                  rm "$HOME"/incomplete
+                fi
+              ''}
+            '';
+          };
+        }; }
+        { "${ctr.name}_scheduler" = {
+          wantedBy = [ "multi-user.target" ];
+          path = with pkgs; [
+            coreutils
+            consul
+            cryptsetup
+            mount
+            util-linux
+            systemd
+            retry
+          ];
+          serviceConfig = let
+            containerDirectory = lib.removeSuffix "/%i" config.systemd.services."container@${ctr.name}".environment.root;
+          in {
+            Restart = "always";
+            RestartSec = "5s";
+            ExecStart = "${pkgs.consul}/bin/consul lock -verbose -monitor-retry 100 container_${ctr.name} ${pkgs.writers.writeBash "${ctr.name}-start" ''
+              set -efux
+              if test -e /var/lib/sync-containers3/${ctr.name}/incomplete; then
+                echo 'data is inconistent, start aborted'
+                exit 1
+              fi
+              trap ${pkgs.writers.writeDash "stop-${ctr.name}" ''
+                set -efux
+                /run/current-system/sw/bin/nixos-container stop ${ctr.name} || :
+                umount /var/lib/sync-containers3/${ctr.name}/state || :
+                cryptsetup luksClose ${ctr.name} || :
+              ''} INT TERM EXIT
+              consul kv put containers/${ctr.name}/host ${config.networking.hostName}
+              cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name}
+              mkdir -p /var/lib/sync-containers3/${ctr.name}/state
+              mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
+              /run/current-system/sw/bin/nixos-container start ${ctr.name}
+              set +x
+              until /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null; do sleep 5; done
+              while retry -t 5 -d 60 -- /run/wrappers/bin/ping -q -c 3 ${ctr.name}.r > /dev/null; do sleep 5; done
+              echo "lost tinc connection to container, shutting down"
+            ''}";
+          };
+        }; }
+      ]) (lib.attrValues cfg.containers)));
+
+      systemd.timers = lib.mapAttrs' (n: ctr: lib.nameValuePair "${ctr.name}_syncer" {
+        timerConfig = {
+          RandomizedDelaySec = 100;
+        };
+      }) cfg.containers;
+
+      users.groups = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" {
+      }) cfg.containers;
+      users.users = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" ({
+        group = "container_${ctr.name}";
+        isNormalUser = true;
+        uid = slib.genid_uint31 "container_${ctr.name}";
+        home = "/var/lib/sync-containers3/${ctr.name}";
+        createHome = true;
+        homeMode = "705";
+      })) cfg.containers;
+
+    })
+    (lib.mkIf (cfg.containers != {}) {
+      # networking
+      networking.networkmanager.unmanaged = [ "ctr0" ];
+      networking.interfaces.dummy0.virtual = true;
+      networking.bridges.ctr0.interfaces = [ "dummy0" ];
+      networking.interfaces.ctr0.ipv4.addresses = [{
+        address = "10.233.0.1";
+        prefixLength = 24;
+      }];
+      systemd.services."dhcpd-ctr0" = {
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        serviceConfig = {
+          Type = "forking";
+          Restart = "always";
+          DynamicUser = true;
+          StateDirectory = "dhcpd-ctr0";
+          User = "dhcpd-ctr0";
+          Group = "dhcpd-ctr0";
+          AmbientCapabilities = [
+           "CAP_NET_RAW"          # to send ICMP messages
+           "CAP_NET_BIND_SERVICE" # to bind on DHCP port (67)
+          ];
+          ExecStartPre = "${pkgs.coreutils}/bin/touch /var/lib/dhcpd-ctr0/dhcpd.leases";
+          ExecStart = "${pkgs.dhcp}/bin/dhcpd -4 -lf /var/lib/dhcpd-ctr0/dhcpd.leases -cf ${pkgs.writeText "dhpd.conf" ''
+            default-lease-time 600;
+            max-lease-time 7200;
+            authoritative;
+            ddns-update-style interim;
+            log-facility local1; # see dhcpd.nix
+
+            option subnet-mask 255.255.255.0;
+            option routers 10.233.0.1;
+            # option domain-name-servers 8.8.8.8; # TODO configure dns server
+            subnet 10.233.0.0 netmask 255.255.255.0 {
+              range 10.233.0.10 10.233.0.250;
+            }
+          ''} ctr0";
+        };
+      };
+    })
+    (lib.mkIf cfg.inContainer.enable {
+      users.groups.container_sync = {};
+      users.users.container_sync = {
+        group = "container_sync";
+        uid = slib.genid_uint31 "container_sync";
+        isNormalUser = true;
+        home = "/var/lib/self";
+        createHome = true;
+        openssh.authorizedKeys.keys = [
+          cfg.inContainer.pubkey
+        ];
+      };
+    })
+  ];
+}
-- 
cgit v1.2.3


From 17fc13592935fc7c8119e24980ea3a83f9773043 Mon Sep 17 00:00:00 2001
From: lassulus <lassulus@lassul.us>
Date: Thu, 17 Nov 2022 11:58:59 +0100
Subject: l sync-containers3: don't kill container on lock loss

---
 lass/3modules/sync-containers3.nix | 124 ++++++++++++++++++++++++++++++-------
 1 file changed, 102 insertions(+), 22 deletions(-)

(limited to 'lass/3modules/sync-containers3.nix')

diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix
index 5b6fa37e1..104b92b24 100644
--- a/lass/3modules/sync-containers3.nix
+++ b/lass/3modules/sync-containers3.nix
@@ -99,7 +99,7 @@ in {
               set -efux
               consul lock sync_${ctr.name} ${pkgs.writers.writeDash "${ctr.name}-sync" ''
                 set -efux
-                if ping -c 1 ${ctr.name}.r; then
+                if /run/wrappers/bin/ping -c 1 ${ctr.name}.r; then
                   touch "$HOME"/incomplete
                   rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --inplace container_sync@${ctr.name}.r:disk "$HOME"/disk
                   rm "$HOME"/incomplete
@@ -108,6 +108,54 @@ in {
             '';
           };
         }; }
+        { "${ctr.name}_watcher" = {
+          path = with pkgs; [
+            coreutils
+            consul
+            cryptsetup
+            curl
+            mount
+            util-linux
+            jq
+          ];
+          serviceConfig = {
+            ExecStart = pkgs.writers.writeDash "${ctr.name}_watcher" ''
+              set -efux
+              while sleep 5; do
+                # get the payload
+                # check if the host reacted recently
+                case $(curl -s -o /dev/null --retry 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in
+                  404)
+                    echo 'got 404 from kv, should kill the container'
+                    break
+                    ;;
+                  500)
+                    echo 'got 500 from kv, will kill container'
+                    break
+                    ;;
+                  200)
+                    # echo 'got 200 from kv, will check payload'
+                    export payload=$(consul kv get containers/${ctr.name})
+                    if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then
+                      # echo 'we are the host, continuing'
+                      continue
+                    else
+                      echo 'we are not host, killing container'
+                      break
+                    fi
+                    ;;
+                  *)
+                    echo 'unknown state, continuing'
+                    continue
+                    ;;
+                esac
+              done
+              /run/current-system/sw/bin/nixos-container stop ${ctr.name} || :
+              umount /var/lib/sync-containers3/${ctr.name}/state || :
+              cryptsetup luksClose ${ctr.name} || :
+            '';
+          };
+        }; }
         { "${ctr.name}_scheduler" = {
           wantedBy = [ "multi-user.target" ];
           path = with pkgs; [
@@ -116,36 +164,68 @@ in {
             cryptsetup
             mount
             util-linux
+            curl
             systemd
+            jq
             retry
+            bc
           ];
-          serviceConfig = let
-            containerDirectory = lib.removeSuffix "/%i" config.systemd.services."container@${ctr.name}".environment.root;
-          in {
+          serviceConfig = {
             Restart = "always";
-            RestartSec = "5s";
-            ExecStart = "${pkgs.consul}/bin/consul lock -verbose -monitor-retry 100 container_${ctr.name} ${pkgs.writers.writeBash "${ctr.name}-start" ''
+            RestartSec = "30s";
+            ExecStart = pkgs.writers.writeDash "${ctr.name}_scheduler" ''
               set -efux
+              # get the payload
+              # check if the host reacted recently
+              case $(curl -s -o /dev/null --retry 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in
+                404)
+                  # echo 'got 404 from kv, will create container'
+                  ;;
+                500)
+                  # echo 'got 500 from kv, retrying again'
+                  exit 0
+                  ;;
+                200)
+                  # echo 'got 200 from kv, will check payload'
+                  export payload=$(consul kv get containers/${ctr.name})
+                  if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then
+                    echo 'we are the host, starting container'
+                  else
+                    # echo 'we are not host, checking timestamp'
+                    # if [ $(echo "$(date +%s) - $(jq -rn 'env.payload | fromjson.time') > 100" | bc) -eq 1 ]; then
+                    if [ "$(jq -rn 'env.payload | fromjson.time | now - tonumber > 100')" = 'true' ]; then
+                      echo 'last beacon is more than 100s ago, taking over'
+                    else
+                      # echo 'last beacon was recent. trying again'
+                      exit 0
+                    fi
+                  fi
+                  ;;
+                *)
+                  echo 'unknown state, bailing out'
+                  exit 0
+                  ;;
+              esac
               if test -e /var/lib/sync-containers3/${ctr.name}/incomplete; then
                 echo 'data is inconistent, start aborted'
                 exit 1
               fi
-              trap ${pkgs.writers.writeDash "stop-${ctr.name}" ''
-                set -efux
-                /run/current-system/sw/bin/nixos-container stop ${ctr.name} || :
-                umount /var/lib/sync-containers3/${ctr.name}/state || :
-                cryptsetup luksClose ${ctr.name} || :
-              ''} INT TERM EXIT
-              consul kv put containers/${ctr.name}/host ${config.networking.hostName}
-              cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name}
-              mkdir -p /var/lib/sync-containers3/${ctr.name}/state
-              mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
-              /run/current-system/sw/bin/nixos-container start ${ctr.name}
-              set +x
-              until /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null; do sleep 5; done
-              while retry -t 5 -d 60 -- /run/wrappers/bin/ping -q -c 3 ${ctr.name}.r > /dev/null; do sleep 5; done
-              echo "lost tinc connection to container, shutting down"
-            ''}";
+              consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null
+              consul lock -verbose -monitor-retry=100 -timeout 30s -name container_${ctr.name} container_${ctr.name} ${pkgs.writers.writeBash "${ctr.name}-start" ''
+                set -efu
+                cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} || :
+                mkdir -p /var/lib/sync-containers3/${ctr.name}/state
+                mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
+                /run/current-system/sw/bin/nixos-container start ${ctr.name}
+                # wait for system to become reachable for the first time
+                retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null
+                systemctl start ${ctr.name}_watcher.service
+                while systemctl is-active container@${ctr.name}.service >/devnull && /run/wrappers/bin/ping -q -c 3 ${ctr.name}.r >/dev/null; do
+                  consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null
+                  sleep 10
+                done
+              ''}
+            '';
           };
         }; }
       ]) (lib.attrValues cfg.containers)));
-- 
cgit v1.2.3


From 953f78184f7fddce66de7a3f6161c9700e51871e Mon Sep 17 00:00:00 2001
From: lassulus <lassulus@lassul.us>
Date: Mon, 21 Nov 2022 23:53:54 +0100
Subject: l sync-containers3: increase kill timeout

---
 lass/3modules/sync-containers3.nix | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

(limited to 'lass/3modules/sync-containers3.nix')

diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix
index 104b92b24..1371d5233 100644
--- a/lass/3modules/sync-containers3.nix
+++ b/lass/3modules/sync-containers3.nix
@@ -117,6 +117,7 @@ in {
             mount
             util-linux
             jq
+            retry
           ];
           serviceConfig = {
             ExecStart = pkgs.writers.writeDash "${ctr.name}_watcher" ''
@@ -124,7 +125,7 @@ in {
               while sleep 5; do
                 # get the payload
                 # check if the host reacted recently
-                case $(curl -s -o /dev/null --retry 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in
+                case $(curl -s -o /dev/null --retry 10 --retry-delay 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in
                   404)
                     echo 'got 404 from kv, should kill the container'
                     break
@@ -137,8 +138,14 @@ in {
                     # echo 'got 200 from kv, will check payload'
                     export payload=$(consul kv get containers/${ctr.name})
                     if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then
-                      # echo 'we are the host, continuing'
-                      continue
+                      # echo 'we are the host, trying to reach container'
+                      if $(retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null); then
+                        # echo 'container is reachable, continueing'
+                        continue
+                      else
+                        # echo 'container seems dead, killing'
+                        break
+                      fi
                     else
                       echo 'we are not host, killing container'
                       break
-- 
cgit v1.2.3