From f76693049d10ae507ae649bf3325c4bd8e536b2a Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 18 Jan 2023 20:05:23 +0100 Subject: l sync-containers3: use atomic and nicer rsync --- lass/3modules/sync-containers3.nix | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 86aa40f03..9e7c4ddfd 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -104,9 +104,8 @@ in { consul lock sync_${ctr.name} ${pkgs.writers.writeDash "${ctr.name}-sync" '' set -efux if /run/wrappers/bin/ping -c 1 ${ctr.name}.r; then - touch "$HOME"/incomplete - rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --timeout=30 --inplace container_sync@${ctr.name}.r:disk "$HOME"/disk - rm "$HOME"/incomplete + nice --adjustment=30 rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --timeout=30 container_sync@${ctr.name}.r:disk "$HOME"/disk + rm -f "$HOME"/incomplete fi ''} ''; @@ -218,10 +217,6 @@ in { exit 0 ;; esac - if test -e /var/lib/sync-containers3/${ctr.name}/incomplete; then - echo 'data is inconistent, start aborted' - exit 1 - fi consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null consul lock -verbose -monitor-retry=100 -timeout 30s -name container_${ctr.name} container_${ctr.name} ${pkgs.writers.writeBash "${ctr.name}-start" '' set -efu -- cgit v1.2.3 From 501330a87c362afa8bd325711f8f3346795be3b0 Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 18 Jan 2023 20:06:29 +0100 Subject: l sync-containers3: start watcher before pinging --- lass/3modules/sync-containers3.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 9e7c4ddfd..03eb14e66 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -225,8 +225,8 @@ in { mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state /run/current-system/sw/bin/nixos-container start ${ctr.name} # wait for system to become reachable for the first time - retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null systemctl start ${ctr.name}_watcher.service + retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null while systemctl is-active container@${ctr.name}.service >/devnull && /run/wrappers/bin/ping -q -c 3 ${ctr.name}.r >/dev/null; do consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null sleep 10 -- cgit v1.2.3 From 4d64e1755794cd8364afce6dbcbea72c04466dfa Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 18 Jan 2023 20:06:48 +0100 Subject: l sync-containers3: remove interface on container shutdown --- lass/3modules/sync-containers3.nix | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 03eb14e66..7966f4097 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -235,6 +235,13 @@ in { ''; }; }; } + { "container@${ctr.name}" = lib.mkIf ctr.runContainer { + serviceConfig = { + ExecStop = pkgs.writers.writeDash "remove_interface" '' + ${pkgs.iproute2}/bin/ip link del vb-${ctr.name} + ''; + }; + }; } ]) (lib.attrValues cfg.containers))); systemd.timers = lib.mapAttrs' (n: ctr: lib.nameValuePair "${ctr.name}_syncer" { -- cgit v1.2.3 From d9d0fbd0406873648c985d2b96d2a52c91f9fc9e Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 18 Jan 2023 20:07:18 +0100 Subject: l sync-containers3: configure NAT more directly --- lass/3modules/sync-containers3.nix | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 7966f4097..8f6f74a3c 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -282,14 +282,19 @@ in { }) (lib.mkIf (cfg.containers != {}) { # networking + + # needed because otherwise we lose local dns + environment.etc."resolv.conf".source = lib.mkForce "/run/systemd/resolve/resolv.conf"; + + boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkDefault 1; systemd.network.networks.ctr0 = { name = "ctr0"; address = [ "10.233.0.1/24" ]; networkConfig = { - IPForward = "yes"; - IPMasquerade = "both"; + # IPForward = "yes"; + # IPMasquerade = "both"; ConfigureWithoutCarrier = true; DHCPServer = "yes"; }; @@ -306,6 +311,9 @@ in { { predicate = "-i ctr0"; target = "ACCEPT"; } { predicate = "-o ctr0"; target = "ACCEPT"; } ]; + krebs.iptables.tables.nat.POSTROUTING.rules = [ + { v6 = false; predicate = "-s 10.233.0.0/24"; target = "MASQUERADE"; } + ]; }) (lib.mkIf cfg.inContainer.enable { users.groups.container_sync = {}; -- cgit v1.2.3 From 347bb9ae00f8f1b6942f94d4c983593052a5c227 Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 18 Jan 2023 23:43:38 +0100 Subject: l sync-containers3: force sysctl value to not conflict --- lass/3modules/sync-containers3.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lass/3modules') diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix index 8f6f74a3c..5188f270d 100644 --- a/lass/3modules/sync-containers3.nix +++ b/lass/3modules/sync-containers3.nix @@ -286,7 +286,7 @@ in { # needed because otherwise we lose local dns environment.etc."resolv.conf".source = lib.mkForce "/run/systemd/resolve/resolv.conf"; - boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkDefault 1; + boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkForce 1; systemd.network.networks.ctr0 = { name = "ctr0"; address = [ -- cgit v1.2.3 From c7417c8bc1b50d466dae493ac3619d9f324f34f8 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 30 Jan 2023 20:53:24 +0100 Subject: sync-containers3: lass -> krebs --- lass/3modules/default.nix | 1 - lass/3modules/sync-containers3.nix | 343 ------------------------------------- 2 files changed, 344 deletions(-) delete mode 100644 lass/3modules/sync-containers3.nix (limited to 'lass/3modules') diff --git a/lass/3modules/default.nix b/lass/3modules/default.nix index 42efa8cd6..3a0b1306c 100644 --- a/lass/3modules/default.nix +++ b/lass/3modules/default.nix @@ -15,6 +15,5 @@ _: ./xjail.nix ./autowifi.nix ./browsers.nix - ./sync-containers3.nix ]; } diff --git a/lass/3modules/sync-containers3.nix b/lass/3modules/sync-containers3.nix deleted file mode 100644 index 5188f270d..000000000 --- a/lass/3modules/sync-containers3.nix +++ /dev/null @@ -1,343 +0,0 @@ -{ config, lib, pkgs, ... }: let - cfg = config.lass.sync-containers3; - slib = pkgs.stockholm.lib; -in { - options.lass.sync-containers3 = { - inContainer = { - enable = lib.mkEnableOption "container config for syncing"; - pubkey = lib.mkOption { - type = lib.types.str; # TODO ssh key - }; - }; - containers = lib.mkOption { - default = {}; - type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: { - options = { - name = lib.mkOption { - type = lib.types.str; - default = config._module.args.name; - }; - sshKey = lib.mkOption { - type = slib.types.absolute-pathname; - }; - luksKey = lib.mkOption { - type = slib.types.absolute-pathname; - default = config.sshKey; - }; - ephemeral = lib.mkOption { - type = lib.types.bool; - default = false; - }; - runContainer = lib.mkOption { - type = lib.types.bool; - default = true; - }; - }; - })); - }; - }; - config = lib.mkMerge [ - (lib.mkIf (cfg.containers != {}) { - - containers = lib.mapAttrs' (n: ctr: lib.nameValuePair ctr.name { - config = { - environment.systemPackages = [ - pkgs.dhcpcd - pkgs.git - pkgs.jq - ]; - networking.useDHCP = lib.mkForce true; - systemd.services.autoswitch = { - environment = { - NIX_REMOTE = "daemon"; - }; - wantedBy = [ "multi-user.target" ]; - serviceConfig.ExecStart = pkgs.writers.writeDash "autoswitch" '' - set -efu - mkdir -p /var/state/var_src - ln -Tfrs /var/state/var_src /var/src - if test -e /var/src/nixos-config; then - /run/current-system/sw/bin/nixos-rebuild -I /var/src switch || : - fi - ''; - unitConfig.X-StopOnRemoval = false; - }; - }; - autoStart = false; - enableTun = true; - ephemeral = ctr.ephemeral; - privateNetwork = true; - hostBridge = "ctr0"; - bindMounts = { - "/var/lib/self/disk" = { - hostPath = "/var/lib/sync-containers3/${ctr.name}/disk"; - isReadOnly = false; - }; - "/var/state" = { - hostPath = "/var/lib/sync-containers3/${ctr.name}/state"; - isReadOnly = false; - }; - }; - }) (lib.filterAttrs (_: ctr: ctr.runContainer) cfg.containers); - - systemd.services = lib.foldr lib.recursiveUpdate {} (lib.flatten (map (ctr: [ - { "${ctr.name}_syncer" = { - path = with pkgs; [ - coreutils - consul - rsync - openssh - systemd - ]; - startAt = "*:0/1"; - serviceConfig = { - User = "${ctr.name}_container"; - LoadCredential = [ - "ssh_key:${ctr.sshKey}" - ]; - ExecCondition = pkgs.writers.writeDash "${ctr.name}_checker" '' - set -efu - ! systemctl is-active --quiet container@${ctr.name}.service - ''; - ExecStart = pkgs.writers.writeDash "${ctr.name}_syncer" '' - set -efux - consul lock sync_${ctr.name} ${pkgs.writers.writeDash "${ctr.name}-sync" '' - set -efux - if /run/wrappers/bin/ping -c 1 ${ctr.name}.r; then - nice --adjustment=30 rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --timeout=30 container_sync@${ctr.name}.r:disk "$HOME"/disk - rm -f "$HOME"/incomplete - fi - ''} - ''; - }; - }; } - { "${ctr.name}_watcher" = lib.mkIf ctr.runContainer { - path = with pkgs; [ - coreutils - consul - cryptsetup - curl - mount - util-linux - jq - retry - ]; - serviceConfig = { - ExecStart = pkgs.writers.writeDash "${ctr.name}_watcher" '' - set -efux - while sleep 5; do - # get the payload - # check if the host reacted recently - case $(curl -s -o /dev/null --retry 10 --retry-delay 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in - 404) - echo 'got 404 from kv, should kill the container' - break - ;; - 500) - echo 'got 500 from kv, will kill container' - break - ;; - 200) - # echo 'got 200 from kv, will check payload' - payload=$(consul kv get containers/${ctr.name}) || continue - export payload - if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then - # echo 'we are the host, trying to reach container' - if $(retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null); then - # echo 'container is reachable, continueing' - continue - else - # echo 'container seems dead, killing' - break - fi - else - echo 'we are not host, killing container' - break - fi - ;; - *) - echo 'unknown state, continuing' - continue - ;; - esac - done - /run/current-system/sw/bin/nixos-container stop ${ctr.name} || : - umount /var/lib/sync-containers3/${ctr.name}/state || : - cryptsetup luksClose ${ctr.name} || : - ''; - }; - }; } - { "${ctr.name}_scheduler" = lib.mkIf ctr.runContainer { - wantedBy = [ "multi-user.target" ]; - path = with pkgs; [ - coreutils - consul - cryptsetup - mount - util-linux - curl - systemd - jq - retry - bc - ]; - serviceConfig = { - Restart = "always"; - RestartSec = "30s"; - ExecStart = pkgs.writers.writeDash "${ctr.name}_scheduler" '' - set -efux - # get the payload - # check if the host reacted recently - case $(curl -s -o /dev/null --retry 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in - 404) - # echo 'got 404 from kv, will create container' - ;; - 500) - # echo 'got 500 from kv, retrying again' - exit 0 - ;; - 200) - # echo 'got 200 from kv, will check payload' - export payload=$(consul kv get containers/${ctr.name}) - if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then - echo 'we are the host, starting container' - else - # echo 'we are not host, checking timestamp' - # if [ $(echo "$(date +%s) - $(jq -rn 'env.payload | fromjson.time') > 100" | bc) -eq 1 ]; then - if [ "$(jq -rn 'env.payload | fromjson.time | now - tonumber > 100')" = 'true' ]; then - echo 'last beacon is more than 100s ago, taking over' - else - # echo 'last beacon was recent. trying again' - exit 0 - fi - fi - ;; - *) - echo 'unknown state, bailing out' - exit 0 - ;; - esac - consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null - consul lock -verbose -monitor-retry=100 -timeout 30s -name container_${ctr.name} container_${ctr.name} ${pkgs.writers.writeBash "${ctr.name}-start" '' - set -efu - cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} || : - mkdir -p /var/lib/sync-containers3/${ctr.name}/state - mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state - /run/current-system/sw/bin/nixos-container start ${ctr.name} - # wait for system to become reachable for the first time - systemctl start ${ctr.name}_watcher.service - retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null - while systemctl is-active container@${ctr.name}.service >/devnull && /run/wrappers/bin/ping -q -c 3 ${ctr.name}.r >/dev/null; do - consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null - sleep 10 - done - ''} - ''; - }; - }; } - { "container@${ctr.name}" = lib.mkIf ctr.runContainer { - serviceConfig = { - ExecStop = pkgs.writers.writeDash "remove_interface" '' - ${pkgs.iproute2}/bin/ip link del vb-${ctr.name} - ''; - }; - }; } - ]) (lib.attrValues cfg.containers))); - - systemd.timers = lib.mapAttrs' (n: ctr: lib.nameValuePair "${ctr.name}_syncer" { - timerConfig = { - RandomizedDelaySec = 100; - }; - }) cfg.containers; - - users.groups = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" { - }) cfg.containers; - users.users = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" ({ - group = "${ctr.name}_container"; - isNormalUser = true; - uid = slib.genid_uint31 "container_${ctr.name}"; - home = "/var/lib/sync-containers3/${ctr.name}"; - createHome = true; - homeMode = "705"; - })) cfg.containers; - - environment.systemPackages = lib.mapAttrsToList (_: ctr: (pkgs.writers.writeDashBin "${ctr.name}_init" '' - set -efux - export PATH=${lib.makeBinPath [ - pkgs.coreutils - pkgs.cryptsetup - pkgs.libxfs.bin - ]}:$PATH - truncate -s 5G /var/lib/sync-containers3/${ctr.name}/disk - cryptsetup luksFormat /var/lib/sync-containers3/${ctr.name}/disk ${ctr.luksKey} - cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} - mkfs.xfs /dev/mapper/${ctr.name} - mkdir -p /var/lib/sync-containers3/${ctr.name}/state - mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state - /run/current-system/sw/bin/nixos-container start ${ctr.name} - /run/current-system/sw/bin/nixos-container run ${ctr.name} -- ${pkgs.writeDash "init" '' - mkdir -p /var/state - ''} - '')) cfg.containers; - }) - (lib.mkIf (cfg.containers != {}) { - # networking - - # needed because otherwise we lose local dns - environment.etc."resolv.conf".source = lib.mkForce "/run/systemd/resolve/resolv.conf"; - - boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkForce 1; - systemd.network.networks.ctr0 = { - name = "ctr0"; - address = [ - "10.233.0.1/24" - ]; - networkConfig = { - # IPForward = "yes"; - # IPMasquerade = "both"; - ConfigureWithoutCarrier = true; - DHCPServer = "yes"; - }; - }; - systemd.network.netdevs.ctr0.netdevConfig = { - Kind = "bridge"; - Name = "ctr0"; - }; - networking.networkmanager.unmanaged = [ "ctr0" ]; - krebs.iptables.tables.filter.INPUT.rules = [ - { predicate = "-i ctr0"; target = "ACCEPT"; } - ]; - krebs.iptables.tables.filter.FORWARD.rules = [ - { predicate = "-i ctr0"; target = "ACCEPT"; } - { predicate = "-o ctr0"; target = "ACCEPT"; } - ]; - krebs.iptables.tables.nat.POSTROUTING.rules = [ - { v6 = false; predicate = "-s 10.233.0.0/24"; target = "MASQUERADE"; } - ]; - }) - (lib.mkIf cfg.inContainer.enable { - users.groups.container_sync = {}; - users.users.container_sync = { - group = "container_sync"; - uid = slib.genid_uint31 "container_sync"; - isNormalUser = true; - home = "/var/lib/self"; - createHome = true; - openssh.authorizedKeys.keys = [ - cfg.inContainer.pubkey - ]; - }; - - networking.useHostResolvConf = false; - networking.useNetworkd = true; - systemd.network = { - enable = true; - networks.eth0 = { - matchConfig.Name = "eth0"; - DHCP = "yes"; - dhcpV4Config.UseDNS = true; - }; - }; - }) - ]; -} -- cgit v1.2.3