diff options
Diffstat (limited to 'krebs')
| -rw-r--r-- | krebs/2configs/reaktor2.nix | 4 | ||||
| -rw-r--r-- | krebs/3modules/default.nix | 1 | ||||
| -rw-r--r-- | krebs/3modules/github/known-hosts.nix | 3 | ||||
| -rwxr-xr-x | krebs/3modules/github/update | 15 | ||||
| -rw-r--r-- | krebs/3modules/sync-containers3.nix | 343 | ||||
| -rw-r--r-- | krebs/3modules/tinc.nix | 18 | ||||
| -rw-r--r-- | krebs/5pkgs/simple/syncthing-device-id.nix | 5 | ||||
| -rw-r--r-- | krebs/nixpkgs-unstable.json | 8 | ||||
| -rw-r--r-- | krebs/nixpkgs.json | 8 |
9 files changed, 380 insertions, 25 deletions
diff --git a/krebs/2configs/reaktor2.nix b/krebs/2configs/reaktor2.nix index 11aaf876a..39039cc11 100644 --- a/krebs/2configs/reaktor2.nix +++ b/krebs/2configs/reaktor2.nix @@ -52,7 +52,7 @@ let }; confuse = { - pattern = "^!confuse (.*)$"; + pattern = "!confuse (.*)$"; activate = "match"; arguments = [1]; command = { @@ -90,7 +90,7 @@ let }; confuse_hackint = { - pattern = "^!confuse (.*)$"; + pattern = "!confuse (.*)$"; activate = "match"; arguments = [1]; command = { diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index bff7e135f..6d763afed 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -53,6 +53,7 @@ let ./sitemap.nix ./ssl.nix ./sync-containers.nix + ./sync-containers3.nix ./systemd.nix ./tinc.nix ./tinc_graphs.nix diff --git a/krebs/3modules/github/known-hosts.nix b/krebs/3modules/github/known-hosts.nix index f2705caa4..c0d0b588a 100644 --- a/krebs/3modules/github/known-hosts.nix +++ b/krebs/3modules/github/known-hosts.nix @@ -3,8 +3,7 @@ hostNames = ["github.com"] ++ - # List generated with (IPv6 addresses are currently ignored): - # curl -sS https://api.github.com/meta | jq -r .git[] | grep -v : | nix-shell -p cidr2glob --run cidr2glob | jq -Rs 'split("\n")|map(select(.!=""))' > known-hosts.json + # update known-hosts.json using ./update lib.importJSON ./known-hosts.json ; publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="; diff --git a/krebs/3modules/github/update b/krebs/3modules/github/update new file mode 100755 index 000000000..3952dabae --- /dev/null +++ b/krebs/3modules/github/update @@ -0,0 +1,15 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p cidr2glob curl git jq + +# update known-hosts.json +# +# usage: ./update + +set -efu + +# XXX IPv6 addresses are currently ignored +curl -sS https://api.github.com/meta | jq -r .git[] | grep -v : | cidr2glob | jq -Rs 'split("\n")|map(select(.!=""))' > known-hosts.json + +if git diff --exit-code known-hosts.json; then + echo known-hosts.json is up to date: nothing to do >&2 +fi diff --git a/krebs/3modules/sync-containers3.nix b/krebs/3modules/sync-containers3.nix new file mode 100644 index 000000000..4a00b23ab --- /dev/null +++ b/krebs/3modules/sync-containers3.nix @@ -0,0 +1,343 @@ +{ config, lib, pkgs, ... }: let + cfg = config.krebs.sync-containers3; + slib = pkgs.stockholm.lib; +in { + options.krebs.sync-containers3 = { + inContainer = { + enable = lib.mkEnableOption "container config for syncing"; + pubkey = lib.mkOption { + type = lib.types.str; # TODO ssh key + }; + }; + containers = lib.mkOption { + default = {}; + type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: { + options = { + name = lib.mkOption { + type = lib.types.str; + default = config._module.args.name; + }; + sshKey = lib.mkOption { + type = slib.types.absolute-pathname; + }; + luksKey = lib.mkOption { + type = slib.types.absolute-pathname; + default = config.sshKey; + }; + ephemeral = lib.mkOption { + type = lib.types.bool; + default = false; + }; + runContainer = lib.mkOption { + type = lib.types.bool; + default = true; + }; + }; + })); + }; + }; + config = lib.mkMerge [ + (lib.mkIf (cfg.containers != {}) { + + containers = lib.mapAttrs' (n: ctr: lib.nameValuePair ctr.name { + config = { + environment.systemPackages = [ + pkgs.dhcpcd + pkgs.git + pkgs.jq + ]; + networking.useDHCP = lib.mkForce true; + systemd.services.autoswitch = { + environment = { + NIX_REMOTE = "daemon"; + }; + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = pkgs.writers.writeDash "autoswitch" '' + set -efu + mkdir -p /var/state/var_src + ln -Tfrs /var/state/var_src /var/src + if test -e /var/src/nixos-config; then + /run/current-system/sw/bin/nixos-rebuild -I /var/src switch || : + fi + ''; + unitConfig.X-StopOnRemoval = false; + }; + }; + autoStart = false; + enableTun = true; + ephemeral = ctr.ephemeral; + privateNetwork = true; + hostBridge = "ctr0"; + bindMounts = { + "/var/lib/self/disk" = { + hostPath = "/var/lib/sync-containers3/${ctr.name}/disk"; + isReadOnly = false; + }; + "/var/state" = { + hostPath = "/var/lib/sync-containers3/${ctr.name}/state"; + isReadOnly = false; + }; + }; + }) (lib.filterAttrs (_: ctr: ctr.runContainer) cfg.containers); + + systemd.services = lib.foldr lib.recursiveUpdate {} (lib.flatten (map (ctr: [ + { "${ctr.name}_syncer" = { + path = with pkgs; [ + coreutils + consul + rsync + openssh + systemd + ]; + startAt = "*:0/1"; + serviceConfig = { + User = "${ctr.name}_container"; + LoadCredential = [ + "ssh_key:${ctr.sshKey}" + ]; + ExecCondition = pkgs.writers.writeDash "${ctr.name}_checker" '' + set -efu + ! systemctl is-active --quiet container@${ctr.name}.service + ''; + ExecStart = pkgs.writers.writeDash "${ctr.name}_syncer" '' + set -efux + consul lock sync_${ctr.name} ${pkgs.writers.writeDash "${ctr.name}-sync" '' + set -efux + if /run/wrappers/bin/ping -c 1 ${ctr.name}.r; then + nice --adjustment=30 rsync -a -e "ssh -i $CREDENTIALS_DIRECTORY/ssh_key" --timeout=30 container_sync@${ctr.name}.r:disk "$HOME"/disk + rm -f "$HOME"/incomplete + fi + ''} + ''; + }; + }; } + { "${ctr.name}_watcher" = lib.mkIf ctr.runContainer { + path = with pkgs; [ + coreutils + consul + cryptsetup + curl + mount + util-linux + jq + retry + ]; + serviceConfig = { + ExecStart = pkgs.writers.writeDash "${ctr.name}_watcher" '' + set -efux + while sleep 5; do + # get the payload + # check if the host reacted recently + case $(curl -s -o /dev/null --retry 10 --retry-delay 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in + 404) + echo 'got 404 from kv, should kill the container' + break + ;; + 500) + echo 'got 500 from kv, will kill container' + break + ;; + 200) + # echo 'got 200 from kv, will check payload' + payload=$(consul kv get containers/${ctr.name}) || continue + export payload + if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then + # echo 'we are the host, trying to reach container' + if $(retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null); then + # echo 'container is reachable, continueing' + continue + else + # echo 'container seems dead, killing' + break + fi + else + echo 'we are not host, killing container' + break + fi + ;; + *) + echo 'unknown state, continuing' + continue + ;; + esac + done + /run/current-system/sw/bin/nixos-container stop ${ctr.name} || : + umount /var/lib/sync-containers3/${ctr.name}/state || : + cryptsetup luksClose ${ctr.name} || : + ''; + }; + }; } + { "${ctr.name}_scheduler" = lib.mkIf ctr.runContainer { + wantedBy = [ "multi-user.target" ]; + path = with pkgs; [ + coreutils + consul + cryptsetup + mount + util-linux + curl + systemd + jq + retry + bc + ]; + serviceConfig = { + Restart = "always"; + RestartSec = "30s"; + ExecStart = pkgs.writers.writeDash "${ctr.name}_scheduler" '' + set -efux + # get the payload + # check if the host reacted recently + case $(curl -s -o /dev/null --retry 10 -w '%{http_code}' http://127.0.0.1:8500/v1/kv/containers/${ctr.name}) in + 404) + # echo 'got 404 from kv, will create container' + ;; + 500) + # echo 'got 500 from kv, retrying again' + exit 0 + ;; + 200) + # echo 'got 200 from kv, will check payload' + export payload=$(consul kv get containers/${ctr.name}) + if [ "$(jq -rn 'env.payload | fromjson.host')" = '${config.networking.hostName}' ]; then + echo 'we are the host, starting container' + else + # echo 'we are not host, checking timestamp' + # if [ $(echo "$(date +%s) - $(jq -rn 'env.payload | fromjson.time') > 100" | bc) -eq 1 ]; then + if [ "$(jq -rn 'env.payload | fromjson.time | now - tonumber > 100')" = 'true' ]; then + echo 'last beacon is more than 100s ago, taking over' + else + # echo 'last beacon was recent. trying again' + exit 0 + fi + fi + ;; + *) + echo 'unknown state, bailing out' + exit 0 + ;; + esac + consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null + consul lock -verbose -monitor-retry=100 -timeout 30s -name container_${ctr.name} container_${ctr.name} ${pkgs.writers.writeBash "${ctr.name}-start" '' + set -efu + cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} || : + mkdir -p /var/lib/sync-containers3/${ctr.name}/state + mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state + /run/current-system/sw/bin/nixos-container start ${ctr.name} + # wait for system to become reachable for the first time + systemctl start ${ctr.name}_watcher.service + retry -t 10 -d 10 -- /run/wrappers/bin/ping -q -c 1 ${ctr.name}.r > /dev/null + while systemctl is-active container@${ctr.name}.service >/devnull && /run/wrappers/bin/ping -q -c 3 ${ctr.name}.r >/dev/null; do + consul kv put containers/${ctr.name} "$(jq -cn '{host: "${config.networking.hostName}", time: now}')" >/dev/null + sleep 10 + done + ''} + ''; + }; + }; } + { "container@${ctr.name}" = lib.mkIf ctr.runContainer { + serviceConfig = { + ExecStop = pkgs.writers.writeDash "remove_interface" '' + ${pkgs.iproute2}/bin/ip link del vb-${ctr.name} + ''; + }; + }; } + ]) (lib.attrValues cfg.containers))); + + systemd.timers = lib.mapAttrs' (n: ctr: lib.nameValuePair "${ctr.name}_syncer" { + timerConfig = { + RandomizedDelaySec = 100; + }; + }) cfg.containers; + + users.groups = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" { + }) cfg.containers; + users.users = lib.mapAttrs' (_: ctr: lib.nameValuePair "${ctr.name}_container" ({ + group = "${ctr.name}_container"; + isNormalUser = true; + uid = slib.genid_uint31 "container_${ctr.name}"; + home = "/var/lib/sync-containers3/${ctr.name}"; + createHome = true; + homeMode = "705"; + })) cfg.containers; + + environment.systemPackages = lib.mapAttrsToList (_: ctr: (pkgs.writers.writeDashBin "${ctr.name}_init" '' + set -efux + export PATH=${lib.makeBinPath [ + pkgs.coreutils + pkgs.cryptsetup + pkgs.libxfs.bin + ]}:$PATH + truncate -s 5G /var/lib/sync-containers3/${ctr.name}/disk + cryptsetup luksFormat /var/lib/sync-containers3/${ctr.name}/disk ${ctr.luksKey} + cryptsetup luksOpen --key-file ${ctr.luksKey} /var/lib/sync-containers3/${ctr.name}/disk ${ctr.name} + mkfs.xfs /dev/mapper/${ctr.name} + mkdir -p /var/lib/sync-containers3/${ctr.name}/state + mountpoint /var/lib/sync-containers3/${ctr.name}/state || mount /dev/mapper/${ctr.name} /var/lib/sync-containers3/${ctr.name}/state + /run/current-system/sw/bin/nixos-container start ${ctr.name} + /run/current-system/sw/bin/nixos-container run ${ctr.name} -- ${pkgs.writeDash "init" '' + mkdir -p /var/state + ''} + '')) cfg.containers; + }) + (lib.mkIf (cfg.containers != {}) { + # networking + + # needed because otherwise we lose local dns + environment.etc."resolv.conf".source = lib.mkForce "/run/systemd/resolve/resolv.conf"; + + boot.kernel.sysctl."net.ipv4.ip_forward" = lib.mkForce 1; + systemd.network.networks.ctr0 = { + name = "ctr0"; + address = [ + "10.233.0.1/24" + ]; + networkConfig = { + # IPForward = "yes"; + # IPMasquerade = "both"; + ConfigureWithoutCarrier = true; + DHCPServer = "yes"; + }; + }; + systemd.network.netdevs.ctr0.netdevConfig = { + Kind = "bridge"; + Name = "ctr0"; + }; + networking.networkmanager.unmanaged = [ "ctr0" ]; + krebs.iptables.tables.filter.INPUT.rules = [ + { predicate = "-i ctr0"; target = "ACCEPT"; } + ]; + krebs.iptables.tables.filter.FORWARD.rules = [ + { predicate = "-i ctr0"; target = "ACCEPT"; } + { predicate = "-o ctr0"; target = "ACCEPT"; } + ]; + krebs.iptables.tables.nat.POSTROUTING.rules = [ + { v6 = false; predicate = "-s 10.233.0.0/24"; target = "MASQUERADE"; } + ]; + }) + (lib.mkIf cfg.inContainer.enable { + users.groups.container_sync = {}; + users.users.container_sync = { + group = "container_sync"; + uid = slib.genid_uint31 "container_sync"; + isNormalUser = true; + home = "/var/lib/self"; + createHome = true; + openssh.authorizedKeys.keys = [ + cfg.inContainer.pubkey + ]; + }; + + networking.useHostResolvConf = false; + networking.useNetworkd = true; + systemd.network = { + enable = true; + networks.eth0 = { + matchConfig.Name = "eth0"; + DHCP = "yes"; + dhcpV4Config.UseDNS = true; + }; + }; + }) + ]; +} diff --git a/krebs/3modules/tinc.nix b/krebs/3modules/tinc.nix index 0babc448a..f6727e4d4 100644 --- a/krebs/3modules/tinc.nix +++ b/krebs/3modules/tinc.nix @@ -125,17 +125,13 @@ with import <stockholm/lib>; hostsPackage = mkOption { type = types.package; - default = pkgs.stdenv.mkDerivation { - name = "${tinc.config.netname}-tinc-hosts"; - phases = [ "installPhase" ]; - installPhase = '' - mkdir $out - ${concatStrings (mapAttrsToList (_: host: '' - echo ${shell.escape host.nets."${tinc.config.netname}".tinc.config} \ - > $out/${shell.escape host.name} - '') tinc.config.hosts)} - ''; - }; + default = + pkgs.write "${tinc.config.netname}-tinc-hosts" + (mapAttrs' + (_: host: (nameValuePair "/${host.name}" { + text = host.nets.${tinc.config.netname}.tinc.config; + })) + tinc.config.hosts); defaultText = "‹netname›-tinc-hosts"; description = '' Package of tinc host configuration files. By default, a package will diff --git a/krebs/5pkgs/simple/syncthing-device-id.nix b/krebs/5pkgs/simple/syncthing-device-id.nix index 9533800fd..74983fc18 100644 --- a/krebs/5pkgs/simple/syncthing-device-id.nix +++ b/krebs/5pkgs/simple/syncthing-device-id.nix @@ -1,12 +1,13 @@ -{ openssl, writePython2Bin }: +{ openssl, writePython3Bin }: -writePython2Bin "syncthing-device-id" { +writePython3Bin "syncthing-device-id" { flakeIgnore = [ "E226" "E302" "E305" "E501" "F401" + "W504" ]; } /* python */ '' import base64 diff --git a/krebs/nixpkgs-unstable.json b/krebs/nixpkgs-unstable.json index 644192bbf..6af475a29 100644 --- a/krebs/nixpkgs-unstable.json +++ b/krebs/nixpkgs-unstable.json @@ -1,9 +1,9 @@ { "url": "https://github.com/NixOS/nixpkgs", - "rev": "64e0bf055f9d25928c31fb12924e59ff8ce71e60", - "date": "2022-12-11T09:33:23+00:00", - "path": "/nix/store/lmiwldi32kcc2qgm68swxgb3xzba0ayc-nixpkgs", - "sha256": "1hmx7hhjr74fqmxhb49yfyrpqhzwayrq48xwjv3a117czpb0gnjx", + "rev": "befc83905c965adfd33e5cae49acb0351f6e0404", + "date": "2023-01-13T18:32:21+01:00", + "path": "/nix/store/bwpp6fchhfw699jn9hsdypyc7ggb72gx-nixpkgs", + "sha256": "0m0ik7z06q3rshhhrg2p0vsrkf2jnqcq5gq1q6wb9g291rhyk6h2", "fetchLFS": false, "fetchSubmodules": false, "deepClone": false, diff --git a/krebs/nixpkgs.json b/krebs/nixpkgs.json index 17bffe634..fd6aeb114 100644 --- a/krebs/nixpkgs.json +++ b/krebs/nixpkgs.json @@ -1,9 +1,9 @@ { "url": "https://github.com/NixOS/nixpkgs", - "rev": "9d692a724e74d2a49f7c985132972f991d144254", - "date": "2022-12-16T13:36:40-05:00", - "path": "/nix/store/76wc0ymx7rw348hpl0bp0yb77sf40xd6-nixpkgs", - "sha256": "1byh49p3kwi6adb1izaalj2ab9disfzq1cx526gwgv20ilmphvnr", + "rev": "2f9fd351ec37f5d479556cd48be4ca340da59b8f", + "date": "2023-01-15T13:38:37-03:00", + "path": "/nix/store/mn2dwzki0d159fl09y87jrvyvcjgyy03-nixpkgs", + "sha256": "0w3ysrhbqhgr1qnh0r9miyqd7yf7vsd4wcd21dffwjlb99lynla8", "fetchLFS": false, "fetchSubmodules": false, "deepClone": false, |