summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorlassulus <lassulus@lassul.us>2022-11-15 15:27:17 +0100
committerlassulus <lassulus@lassul.us>2022-11-15 15:48:33 +0100
commit3aae16c8476602f39163b4a786f95ebd8806c71b (patch)
tree18d023dc3513152f426b858d201ea7d63bb80243
parentc9e353d1aaa5d3331dc678a6181de8d96b5b1541 (diff)
l: init sync-containers3
-rw-r--r--lass/3modules/default.nix1
-rw-r--r--lass/3modules/sync-containers3.nix226
2 files changed, 227 insertions, 0 deletions
diff --git a/lass/3modules/default.nix b/lass/3modules/default.nix
index 3a0b1306c..42efa8cd6 100644
--- a/lass/3modules/default.nix
+++ b/lass/3modules/default.nix
@@ -15,5 +15,6 @@ _:
./xjail.nix
./autowifi.nix
./browsers.nix
+ ./sync-containers3.nix
];
}
diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix
new file mode 100644
index 000000000..5b6fa37e1
--- /dev/null
+++ b/lass/3modules/sync-containers3.nix
@@ -0,0 +1,226 @@
+{ config, lib, pkgs, ... }: let
+ cfg = config.lass.sync-containers3;
+ slib = pkgs.stockholm.lib;
+in {
+ options.lass.sync-containers3 = {
+ inContainer = {
+ enable = lib.mkEnableOption "container config for syncing";
+ pubkey = lib.mkOption {
+ type = lib.types.str; # TODO ssh key
+ };
+ };
+ containers = lib.mkOption {
+ default = {};
+ type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: {
+ options = {
+ name = lib.mkOption {
+ type = lib.types.str;
+ default = config._module.args.name;
+ };
+ sshKey = lib.mkOption {
+ type = slib.types.absolute-pathname;
+ };
+ luksKey = lib.mkOption {
+ type = slib.types.absolute-pathname;
+ default = config.sshKey;
+ };
+ ephemeral = lib.mkOption {
+ type = lib.types.bool;
+ default = false;
+ };
+ };
+ }));
+ };
+ };
+ config = lib.mkMerge [
+ (lib.mkIf (cfg.containers != {}) {
+
+ containers = lib.mapAttrs' (n: ctr: lib.nameValuePair ctr.name {
+ config = {
+ environment.systemPackages = [
+ pkgs.dhcpcd
+ pkgs.git
+ pkgs.jq
+ ];
+ networking.useDHCP = lib.mkForce true;
+ systemd.services.autoswitch = {
+ environment = {
+ NIX_REMOTE = "daemon";
+ };
+ wantedBy = [ "multi-user.target" ];
+ serviceConfig.ExecStart = pkgs.writers.writeDash "autoswitch" ''
+ set -efu
+ ln -frs /var/state/var_src /var/src
+ if test -e /var/src/nixos-config; then
+ /run/current-system/sw/bin/nixos-rebuild -I /var/src switch || :
+ fi
+ '';
+ unitConfig.X-StopOnRemoval = false;
+ };
+ };
+ autoStart = false;
+ enableTun = true;
+ ephemeral = ctr.ephemeral;
+ privateNetwork = true;
+ hostBridge = "ctr0";
+ bindMounts = {
+ "/etc/resolv.conf".hostPath = "/etc/resolv.conf";
+ "/var/lib/self/disk" = {
+ hostPath = "/var/lib/sync-containers3/${ctr.name}/disk";
+ isReadOnly = false;
+ };
+ "/var/state" = {
+ hostPath = "/var/lib/sync-containers3/${ctr.name}/state";
+ isReadOnly = false;
+ };
+ };
+ }) cfg.containers;
+
+ systemd.services = lib.foldr lib.recursiveUpdate {} (lib.flatten (map (ctr: [
+ { "${ctr.name}_syncer" = {
+ path = with pkgs; [
+ coreutils
+ consul
+ rsync
+ openssh
+ systemd
+ ];
+ startAt = "*:0/1";
+ serviceConfig = {
+ User = "${ctr.name}_container";
+ LoadCredential = [
+ "ssh_key:${ctr.sshKey}"
+ ];
+ ExecCondition = pkgs.writers.writeDash "${ctr.name}_checker" ''
+ set -efu
+ ! systemctl is-active --quiet container@${ctr.name}.service
+ '';
+ ExecStart = pkgs.writers.writeDash "${ctr.name}_syncer" ''
+ set -efux
+ consul lock sync_${ctr.name} ${pkgs.writers.writeDash "${ctr.name}-sync" ''
+ set -efux
+ if ping -c 1 ${ctr.name}.r; then
+ touch "$HOME"/incomplete
+ rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --inplace container_sync@${ctr.name}.r:disk "$HOME"/disk
+ rm "$HOME"/incomplete
+ fi
+ ''}
+ '';
+ };
+ }; }
+ { "${ctr.name}_scheduler" = {
+ wantedBy = [ "multi-user.target" ];
+ path = with pkgs; [
+ coreutils
+ consul
+ cryptsetup
+ mount
+ util-linux
+ systemd
+ retry
+ ];
+ serviceConfig = let
+ containerDirectory = lib.removeSuffix "/%i" config.systemd.services."container@${ctr.name}".environment.root;
+ in {
+ Restart = "always";
+ RestartSec = "5s";
+ ExecStart = "${pkgs.consul}/bin/consul lock -verbose -monitor-retry 100 container_${ctr.name} ${pkgs.writers.writeBash "${ctr.name}-start" ''
+ set -efux
+ if test -e /var/lib/sync-containers3/${ctr.name}/incomplete; then
+ echo 'data is inconistent, start aborted'
+ exit 1
+ fi
+ trap ${pkgs.writers.writeDash "stop-${ctr.name}" ''
+ set -efux
+ /run/current-system/sw/bin/nixos-container stop ${ctr.name} || :
+ umount /var/lib/sync-containers3/${ctr.name}/state || :
+ cryptsetup luksClose ${ctr.name} || :
+ ''} INT TERM EXIT
+ consul kv put containers/${ctr.name}/host ${config.networking.hostName}
+ cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name}
+ mkdir -p /var/lib/sync-containers3/${ctr.name}/state
+ mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state
+ /run/current-system/sw/bin/nixos-container start ${ctr.name}
+ set +x
+ until /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null; do sleep 5; done
+ while retry -t 5 -d 60 -- /run/wrappers/bin/ping -q -c 3 ${ctr.name}.r > /dev/null; do sleep 5; done
+ echo "lost tinc connection to container, shutting down"
+ ''}";
+ };
+ }; }
+ ]) (lib.attrValues cfg.containers)));
+
+ systemd.timers = lib.mapAttrs' (n: ctr: lib.nameValuePair "${ctr.name}_syncer" {
+ timerConfig = {
+ RandomizedDelaySec = 100;
+ };
+ }) cfg.containers;
+
+ users.groups = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" {
+ }) cfg.containers;
+ users.users = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" ({
+ group = "container_${ctr.name}";
+ isNormalUser = true;
+ uid = slib.genid_uint31 "container_${ctr.name}";
+ home = "/var/lib/sync-containers3/${ctr.name}";
+ createHome = true;
+ homeMode = "705";
+ })) cfg.containers;
+
+ })
+ (lib.mkIf (cfg.containers != {}) {
+ # networking
+ networking.networkmanager.unmanaged = [ "ctr0" ];
+ networking.interfaces.dummy0.virtual = true;
+ networking.bridges.ctr0.interfaces = [ "dummy0" ];
+ networking.interfaces.ctr0.ipv4.addresses = [{
+ address = "10.233.0.1";
+ prefixLength = 24;
+ }];
+ systemd.services."dhcpd-ctr0" = {
+ wantedBy = [ "multi-user.target" ];
+ after = [ "network.target" ];
+ serviceConfig = {
+ Type = "forking";
+ Restart = "always";
+ DynamicUser = true;
+ StateDirectory = "dhcpd-ctr0";
+ User = "dhcpd-ctr0";
+ Group = "dhcpd-ctr0";
+ AmbientCapabilities = [
+ "CAP_NET_RAW" # to send ICMP messages
+ "CAP_NET_BIND_SERVICE" # to bind on DHCP port (67)
+ ];
+ ExecStartPre = "${pkgs.coreutils}/bin/touch /var/lib/dhcpd-ctr0/dhcpd.leases";
+ ExecStart = "${pkgs.dhcp}/bin/dhcpd -4 -lf /var/lib/dhcpd-ctr0/dhcpd.leases -cf ${pkgs.writeText "dhpd.conf" ''
+ default-lease-time 600;
+ max-lease-time 7200;
+ authoritative;
+ ddns-update-style interim;
+ log-facility local1; # see dhcpd.nix
+
+ option subnet-mask 255.255.255.0;
+ option routers 10.233.0.1;
+ # option domain-name-servers 8.8.8.8; # TODO configure dns server
+ subnet 10.233.0.0 netmask 255.255.255.0 {
+ range 10.233.0.10 10.233.0.250;
+ }
+ ''} ctr0";
+ };
+ };
+ })
+ (lib.mkIf cfg.inContainer.enable {
+ users.groups.container_sync = {};
+ users.users.container_sync = {
+ group = "container_sync";
+ uid = slib.genid_uint31 "container_sync";
+ isNormalUser = true;
+ home = "/var/lib/self";
+ createHome = true;
+ openssh.authorizedKeys.keys = [
+ cfg.inContainer.pubkey
+ ];
+ };
+ })
+ ];
+}