21 files changed, 361 insertions, 603 deletions
diff --git a/krebs/1systems/puyak/config.nix b/krebs/1systems/puyak/config.nix
index 6493c6df4..a20f6929e 100644
--- a/krebs/1systems/puyak/config.nix
+++ b/krebs/1systems/puyak/config.nix
@@ -13,6 +13,7 @@
     <stockholm/krebs/2configs/ircd.nix>
     <stockholm/krebs/2configs/news.nix>
     <stockholm/krebs/2configs/news-spam.nix>
+    <stockholm/krebs/2configs/shack/ssh-keys.nix>
     <stockholm/krebs/2configs/shack/prometheus/node.nix>
     <stockholm/krebs/2configs/shack/prometheus/server.nix>
     <stockholm/krebs/2configs/shack/prometheus/unifi.nix>
@@ -81,12 +82,6 @@
     echo level disengaged > /proc/acpi/ibm/fan
   '';
 
-  # to access vorstand vm
-  users.users.root.openssh.authorizedKeys.keys = [
-    config.krebs.users.ulrich.pubkey
-    config.krebs.users.raute.pubkey
-  ];
-
   users.users.joerg = {
     openssh.authorizedKeys.keys = [ config.krebs.users.Mic92.pubkey ];
     isNormalUser = true;
diff --git a/krebs/1systems/wolf/config.nix b/krebs/1systems/wolf/config.nix
index e87b7bb99..059e09ac1 100644
--- a/krebs/1systems/wolf/config.nix
+++ b/krebs/1systems/wolf/config.nix
@@ -16,6 +16,7 @@ in
 
     # handle the worlddomination map via coap
     <stockholm/krebs/2configs/shack/worlddomination.nix>
+    <stockholm/krebs/2configs/shack/ssh-keys.nix>
 
     # drivedroid.shack for shackphone
     <stockholm/krebs/2configs/shack/drivedroid.nix>
@@ -117,14 +118,6 @@ in
 
   fileSystems."/" = { device = "/dev/disk/by-label/nixos"; fsType = "ext4"; };
 
-  users.extraUsers.root.openssh.authorizedKeys.keys = [
-    config.krebs.users."0x4A6F".pubkey
-    config.krebs.users.ulrich.pubkey
-    config.krebs.users.raute.pubkey
-    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDb9NPa2Hf51afcG1H13UPbE5E02J8aC9a1sGCRls592wAVlQbmojYR1jWDPA2m32Bsyv0ztqi81zDyndWWZPQVJVBk00VjYBcgk6D5ifqoAuWLzfuHJPWZGOvBf/U74/LNFNUkj1ywjneK7HYTRPXrRBBfBSQNmQzkvue7s599L2vdueZKyjNsMpx2m6nm2SchaMuDskSQut/168JgU1l4M8BeT68Bo4WdelhBYnhSI1a59FGkgdu2SCjyighLQRy2sOH3ksnkHWENPkA+wwQOlKl7R3DsEybrNd4NU9FSwFDyDmdhfv5gJp8UGSFdjAwx43+8zM5t5ruZ25J0LnVb0PuTuRA00UsW83MkLxFpDQLrQV08tlsY6iGrqxP67C3VJ6t4v6oTp7/vaRLhEFc1PhOLh+sZ18o8MLO+e2rGmHGHQnSKfBOLUvDMGa4jb01XBGjdnIXLOkVo79YR5jZn7jJb2gTZ95OD6bWSDADoURSuwuLa7kh4ti1ItAKuhkIvbuky3rRVvQEc92kJ6aNUswIUXJa0K2ibbIY6ycKAA3Ljksl3Mm9KzOn6yc/i/lSF+SOrTGhabPJigKkIoqKIwnV5IU3gkfsxPQJOBMPqHDGAOeYQe3WpWedEPYuhQEczw4exMb9TkNE96F71PzuQPJDl5sPAWyPLeMKpy5XbfRiF2by4nxN3ZIQvjtoyVkjNV+qM0q0yKBzLxuRAEQOZ2yCEaBudZQkQiwHD97H2vu4SRQ/2aOie1XiOnmdbQRDZSO3BsoDK569K1w+gDfSnqY7zVUMj6tw+uKx6Gstck5lbvYMtdWKsfPv/pDM8eyIVFLL93dKTX+ertcQj6xDwLfOiNubE5ayFXhYkjwImV6NgfBuq+3hLK0URP2rPlOZbbZTQ0WlKD6CCRZPMSZCU9oD2zYfqpvRArBUcdkAwGePezORkfJQLE6mYEJp6pdFkJ/IeFLbO6M0lZVlfnpzAC9kjjkMCRofZUETcFSppyTImCbgo3+ok59/PkNU5oavBXyW80ue2tWHr08HX/QALNte3UITmIIlU6SFMCPMWJqadK1eDPWfJ4H4iDXRNn3D5wqN++iMloKvpaj0wieqXLY4+YfvNTNr177OU48GEWW8DnoEkbpwsCbjPxznGDQhdDqdYyMY/fDgRQReKITvKYGHRzesGysw5cKsp9LEfXD0R6WE2TeiiENla5AWzTgXJB0AyZEcOiIfqOgT9Nr9S8q5gc/BdA7P+jhGGJgEHhV3dVlfIZ7pmZc27Yu7UTQ0lbAKWqcMSTOdne+QL6ILzbvLrQwdvax4tQdm5opfU16SrOox1AMwAbkdq84z6uJqYVx3cUXfMJgTyDNrVv3or root@plattenschwein" # for backup
-    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1Lx5MKtVjB/Ef6LpEiIAgVwY5xKQFdHuLQR+odQO4cAgxj1QaIXGN0moixY52DebVQhAtiCNiFZ83uJyOj8kmu30yuXwtSOQeqziA859qMJKZ4ZcYdKvbXwnf2Chm5Ck/0FvtpjTWHIZAogwP1wQto/lcqHOjrTAnZeJfQuHTswYUSnmUU5zdsEZ9HidDPUc2Gv0wkBNd+KMQyOZl0HkaxHWvn0h4KK4hYZisOpeTfXJxD87bo+Eg4LL2vvnHW6dF6Ygrbd/0XRMsRRI8OAReVBUoJn7IE1wwAl/FpblNmhaF9hlL7g7hR1ADvaWMMw0e8SSzW6Y+oIa8qFQL6wR1 gitlab-builder" # for being deployed by gitlab ci
-  ];
-
   services.udev.extraRules = ''
     SUBSYSTEM=="net", ATTR{address}=="${external-mac}", NAME="${ext-if}"
   '';
@@ -137,6 +130,7 @@ in
     enable = true;
     wideArea = false;
   };
+
   environment.systemPackages = [ pkgs.avahi ];
 
 }
diff --git a/krebs/2configs/exim-smarthost.nix b/krebs/2configs/exim-smarthost.nix
index 698e20da1..224a38ac3 100644
--- a/krebs/2configs/exim-smarthost.nix
+++ b/krebs/2configs/exim-smarthost.nix
@@ -15,13 +15,12 @@ in {
         makefu
         tv
       ];
-      eloop-ml = spam-ml ++ [ ciko ];
+      eloop-ml = spam-ml;
       spam-ml = [
         lass
         makefu
         tv
       ];
-      ciko.mail = "ciko@slash16.net";
     in {
       "anmeldung@eloop.org" = eloop-ml;
       "brain@krebsco.de" = brain-ml;
diff --git a/krebs/2configs/shack/prometheus/alert-rules.nix b/krebs/2configs/shack/prometheus/alert-rules.nix
new file mode 100644
index 000000000..096c551ba
--- /dev/null
+++ b/krebs/2configs/shack/prometheus/alert-rules.nix
@@ -0,0 +1,102 @@
+{ lib }:
+with lib;
+
+let
+  deviceFilter = ''device!="ramfs",device!="rpc_pipefs",device!="lxcfs",device!="nsfs",device!="borgfs"'';
+in mapAttrsToList (name: opts: {
+  alert = name;
+  expr = opts.condition;
+  for = opts.time or "2m";
+  labels = if (opts.page or true) then { severity = "page"; } else {};
+  annotations = {
+    summary = opts.summary;
+    description = opts.description;
+  };
+}) {
+  node_down = {
+    condition = ''up{job="node"} == 0'';
+    summary = "{{$labels.alias}}: Node is down.";
+    description = "{{$labels.alias}} has been down for more than 2 minutes.";
+  };
+  node_systemd_service_failed = {
+    condition = ''node_systemd_unit_state{state="failed"} == 1'';
+    summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start.";
+    description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}.";
+  };
+  node_filesystem_full_80percent = {
+    condition = ''sort(node_filesystem_free_bytes{${deviceFilter}} < node_filesystem_size_bytes{${deviceFilter}} * 0.2) / 1024^3'';
+    time = "10m";
+    summary = "{{$labels.alias}}: Filesystem is running out of space soon.";
+    description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 20% space left on its filesystem.";
+  };
+  node_filesystem_full_in_7d = {
+    condition = ''predict_linear(node_filesystem_free_bytes{${deviceFilter}}[2d], 7*24*3600) <= 0'';
+    time = "1h";
+    summary = "{{$labels.alias}}: Filesystem is running out of space in 7 days.";
+    description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 7 days";
+  };
+  node_filesystem_full_in_30d = {
+    condition = ''predict_linear(node_filesystem_free_bytes{${deviceFilter}}[30d], 30*24*3600) <= 0'';
+    time = "1h";
+    summary = "{{$labels.alias}}: Filesystem is running out of space in 30 days.";
+    description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 30 days";
+  };
+  node_filedescriptors_full_in_3h = {
+    condition = ''predict_linear(node_filefd_allocated[3h], 3*3600) >= node_filefd_maximum'';
+    time = "20m";
+    summary = "{{$labels.alias}} is running out of available file descriptors in 3 hours.";
+    description = "{{$labels.alias}} is running out of available file descriptors in approx. 3 hours";
+  };
+  node_filedescriptors_full_in_7d = {
+    condition = ''predict_linear(node_filefd_allocated[7d], 7*24*3600) >= node_filefd_maximum'';
+    time = "1h";
+    summary = "{{$labels.alias}} is running out of available file descriptors in 7 days.";
+    description = "{{$labels.alias}} is running out of available file descriptors in approx. 7 days";
+  };
+  node_load15 = {
+    condition = ''node_load15 / on(alias) count(node_cpu_seconds_total{mode="system"}) by (alias) >= 1.0'';
+    time = "10m";
+    summary = "{{$labels.alias}}: Running on high load: {{$value}}";
+    description = "{{$labels.alias}} is running with load15 > 1 for at least 5 minutes: {{$value}}";
+  };
+  node_ram_using_90percent = {
+    condition =  "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1";
+    time = "1h";
+    summary = "{{$labels.alias}}: Using lots of RAM.";
+    description = "{{$labels.alias}} is using at least 90% of its RAM for at least 1 hour.";
+  };
+  node_swap_using_30percent = {
+    condition = "node_memory_SwapTotal_bytes - (node_memory_SwapFree_bytes + node_memory_SwapCached_bytes) > node_memory_SwapTotal_bytes * 0.3";
+    time = "30m";
+    summary = "{{$labels.alias}}: Using more than 30% of its swap.";
+    description = "{{$labels.alias}} is using 30% of its swap space for at least 30 minutes.";
+  };
+  node_visible_confluence_space = {
+    condition = "node_visible_confluence_space != 0";
+    summary = "crowd prometheus cann see the {{$labels.space_name}} confluence space!";
+    description = "crowd user `prometheus` can see the `{{$labels.space_name}}` confluence space.";
+  };
+  node_hwmon_temp = {
+    condition = "node_hwmon_temp_celsius > node_hwmon_temp_crit_celsius*0.9 OR node_hwmon_temp_celsius > node_hwmon_temp_max_celsius*0.95";
+    time = "5m";
+    summary = "{{$labels.alias}}: Sensor {{$labels.sensor}}/{{$labels.chip}} temp is high: {{$value}} ";
+    description = "{{$labels.alias}} reports hwmon sensor {{$labels.sensor}}/{{$labels.chip}} temperature value is nearly critical: {{$value}}";
+  };
+  node_conntrack_limit = {
+    condition  = "node_nf_conntrack_entries_limit - node_nf_conntrack_entries < 1000";
+    time = "5m";
+    summary = "{{$labels.alias}}: Number of tracked connections high";
+    description = "{{$labels.alias}} has only {{$value}} free slots for connection tracking available.";
+  };
+  node_reboot = {
+    condition = "time() - node_boot_time_seconds < 300";
+    summary = "{{$labels.alias}}: Reboot";
+    description = "{{$labels.alias}} just rebooted.";
+  };
+  node_uptime = {
+    condition = "time() - node_boot_time_seconds > 2592000";
+    page = false;
+    summary = "{{$labels.alias}}: Uptime monster";
+    description = "{{$labels.alias}} has been up for more than 30 days.";
+  };
+}
diff --git a/krebs/2configs/shack/prometheus/server.nix b/krebs/2configs/shack/prometheus/server.nix
index 7f6f38610..f5d2e7640 100644
--- a/krebs/2configs/shack/prometheus/server.nix
+++ b/krebs/2configs/shack/prometheus/server.nix
@@ -28,90 +28,12 @@
         "-storage.local.index-cache-size.label-name-to-label-values 2097152"
         "-storage.local.index-cache-size.label-pair-to-fingerprints 41943040"
       ];
-      rules = [
-        ''
-          ALERT node_down
-          IF up == 0
-          FOR 5m
-          LABELS {
-            severity="page"
-          }
-          ANNOTATIONS {
-            summary = "{{$labels.alias}}: Node is down.",
-            description = "{{$labels.alias}} has been down for more than 5 minutes."
-          }
-          ALERT node_systemd_service_failed
-          IF node_systemd_unit_state{state="failed"} == 1
-          FOR 4m
-          LABELS {
-            severity="page"
-          }
-          ANNOTATIONS {
-            summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start.",
-            description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}."
-          }
-          ALERT node_filesystem_full_90percent
-          IF sort(node_filesystem_free{device!="ramfs"} < node_filesystem_size{device!="ramfs"} * 0.1) / 1024^3
-          FOR 5m
-          LABELS {
-            severity="page"
-          }
-          ANNOTATIONS {
-            summary = "{{$labels.alias}}: Filesystem is running out of space soon.",
-            description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% space left on its filesystem."
-          }
-          ALERT node_filesystem_full_in_4h
-          IF predict_linear(node_filesystem_free{device!="ramfs"}[1h], 4*3600) <= 0
-          FOR 5m
-          LABELS {
-            severity="page"
-          }
-          ANNOTATIONS {
-            summary = "{{$labels.alias}}: Filesystem is running out of space in 4 hours.",
-            description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 4 hours"
-          }
-          ALERT node_filedescriptors_full_in_3h
-          IF predict_linear(node_filefd_allocated[1h], 3*3600) >= node_filefd_maximum
-          FOR 20m
-          LABELS {
-            severity="page"
-          }
-          ANNOTATIONS {
-            summary = "{{$labels.alias}} is running out of available file descriptors in 3 hours.",
-            description = "{{$labels.alias}} is running out of available file descriptors in approx. 3 hours"
-          }
-          ALERT node_load1_90percent
-          IF node_load1 / on(alias) count(node_cpu{mode="system"}) by (alias) >= 0.9
-          FOR 1h
-          LABELS {
-            severity="page"
-          }
-          ANNOTATIONS {
-            summary = "{{$labels.alias}}: Running on high load.",
-            description = "{{$labels.alias}} is running with > 90% total load for at least 1h."
-          }
-          ALERT node_cpu_util_90percent
-          IF 100 - (avg by (alias) (irate(node_cpu{mode="idle"}[5m])) * 100) >= 90
-          FOR 1h
-          LABELS {
-            severity="page"
-          }
-          ANNOTATIONS {
-            summary = "{{$labels.alias}}: High CPU utilization.",
-            description = "{{$labels.alias}} has total CPU utilization over 90% for at least 1h."
-          }
-          ALERT node_ram_using_90percent
-          IF node_memory_MemFree + node_memory_Buffers + node_memory_Cached < node_memory_MemTotal * 0.1
-          FOR 30m
-          LABELS {
-            severity="page"
-          }
-          ANNOTATIONS {
-            summary="{{$labels.alias}}: Using lots of RAM.",
-            description="{{$labels.alias}} is using at least 90% of its RAM for at least 30 minutes now.",
-          }
-        ''
-      ];
+      ruleFiles = lib.singleton (pkgs.writeText "prometheus-rules.yml" (builtins.toJSON {
+            groups = lib.singleton {
+              name = "mf-alerting-rules";
+              rules = import ./alert-rules.nix { inherit lib; };
+            };
+          }));
       scrapeConfigs = [
         {
           job_name = "node";
diff --git a/krebs/2configs/shack/ssh-keys.nix b/krebs/2configs/shack/ssh-keys.nix
new file mode 100644
index 000000000..9c7f507f1
--- /dev/null
+++ b/krebs/2configs/shack/ssh-keys.nix
@@ -0,0 +1,10 @@
+{ config, ... }:
+{
+  users.users.root.openssh.authorizedKeys.keys = [
+    config.krebs.users."0x4A6F".pubkey
+    config.krebs.users.ulrich.pubkey
+    config.krebs.users.raute.pubkey
+    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEAQDb9NPa2Hf51afcG1H13UPbE5E02J8aC9a1sGCRls592wAVlQbmojYR1jWDPA2m32Bsyv0ztqi81zDyndWWZPQVJVBk00VjYBcgk6D5ifqoAuWLzfuHJPWZGOvBf/U74/LNFNUkj1ywjneK7HYTRPXrRBBfBSQNmQzkvue7s599L2vdueZKyjNsMpx2m6nm2SchaMuDskSQut/168JgU1l4M8BeT68Bo4WdelhBYnhSI1a59FGkgdu2SCjyighLQRy2sOH3ksnkHWENPkA+wwQOlKl7R3DsEybrNd4NU9FSwFDyDmdhfv5gJp8UGSFdjAwx43+8zM5t5ruZ25J0LnVb0PuTuRA00UsW83MkLxFpDQLrQV08tlsY6iGrqxP67C3VJ6t4v6oTp7/vaRLhEFc1PhOLh+sZ18o8MLO+e2rGmHGHQnSKfBOLUvDMGa4jb01XBGjdnIXLOkVo79YR5jZn7jJb2gTZ95OD6bWSDADoURSuwuLa7kh4ti1ItAKuhkIvbuky3rRVvQEc92kJ6aNUswIUXJa0K2ibbIY6ycKAA3Ljksl3Mm9KzOn6yc/i/lSF+SOrTGhabPJigKkIoqKIwnV5IU3gkfsxPQJOBMPqHDGAOeYQe3WpWedEPYuhQEczw4exMb9TkNE96F71PzuQPJDl5sPAWyPLeMKpy5XbfRiF2by4nxN3ZIQvjtoyVkjNV+qM0q0yKBzLxuRAEQOZ2yCEaBudZQkQiwHD97H2vu4SRQ/2aOie1XiOnmdbQRDZSO3BsoDK569K1w+gDfSnqY7zVUMj6tw+uKx6Gstck5lbvYMtdWKsfPv/pDM8eyIVFLL93dKTX+ertcQj6xDwLfOiNubE5ayFXhYkjwImV6NgfBuq+3hLK0URP2rPlOZbbZTQ0WlKD6CCRZPMSZCU9oD2zYfqpvRArBUcdkAwGePezORkfJQLE6mYEJp6pdFkJ/IeFLbO6M0lZVlfnpzAC9kjjkMCRofZUETcFSppyTImCbgo3+ok59/PkNU5oavBXyW80ue2tWHr08HX/QALNte3UITmIIlU6SFMCPMWJqadK1eDPWfJ4H4iDXRNn3D5wqN++iMloKvpaj0wieqXLY4+YfvNTNr177OU48GEWW8DnoEkbpwsCbjPxznGDQhdDqdYyMY/fDgRQReKITvKYGHRzesGysw5cKsp9LEfXD0R6WE2TeiiENla5AWzTgXJB0AyZEcOiIfqOgT9Nr9S8q5gc/BdA7P+jhGGJgEHhV3dVlfIZ7pmZc27Yu7UTQ0lbAKWqcMSTOdne+QL6ILzbvLrQwdvax4tQdm5opfU16SrOox1AMwAbkdq84z6uJqYVx3cUXfMJgTyDNrVv3or root@plattenschwein" # for backup
+    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1Lx5MKtVjB/Ef6LpEiIAgVwY5xKQFdHuLQR+odQO4cAgxj1QaIXGN0moixY52DebVQhAtiCNiFZ83uJyOj8kmu30yuXwtSOQeqziA859qMJKZ4ZcYdKvbXwnf2Chm5Ck/0FvtpjTWHIZAogwP1wQto/lcqHOjrTAnZeJfQuHTswYUSnmUU5zdsEZ9HidDPUc2Gv0wkBNd+KMQyOZl0HkaxHWvn0h4KK4hYZisOpeTfXJxD87bo+Eg4LL2vvnHW6dF6Ygrbd/0XRMsRRI8OAReVBUoJn7IE1wwAl/FpblNmhaF9hlL7g7hR1ADvaWMMw0e8SSzW6Y+oIa8qFQL6wR1 gitlab-builder" # for being deployed by gitlab ci
+  ];
+}
diff --git a/krebs/3modules/bepasty-server.nix b/krebs/3modules/bepasty-server.nix
index 4892a8723..ffa9a29e9 100644
--- a/krebs/3modules/bepasty-server.nix
+++ b/krebs/3modules/bepasty-server.nix
@@ -2,10 +2,10 @@
 
 with import <stockholm/lib>;
 let
-  gunicorn = pkgs.python27Packages.gunicorn;
-  bepasty = pkgs.bepasty.override { python3Packages = pkgs.python27Packages; };
-  gevent = pkgs.python27Packages.gevent;
-  python = pkgs.python27Packages.python;
+  gunicorn = pkgs.python3Packages.gunicorn;
+  bepasty = pkgs.bepasty;
+  gevent = pkgs.python3Packages.gevent;
+  python = pkgs.python3Packages.python;
   cfg = config.krebs.bepasty;
 
   out = {
diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix
index cbf24effe..7695667fd 100644
--- a/krebs/3modules/ci.nix
+++ b/krebs/3modules/ci.nix
@@ -135,6 +135,7 @@ let
           f_${name} = util.BuildFactory()
           f_${name}.addStep(steps.Git(
               repourl=util.Property('repository', '${head repo.urls}'),
+              method='clobber',
               mode='full',
               submodules=True,
           ))
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index c770391c7..fcdbcbc19 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -103,7 +103,6 @@ let
     { krebs = import ./krebs  { inherit config; }; }
     { krebs = import ./lass   { inherit config; }; }
     { krebs = import ./makefu { inherit config; }; }
-    { krebs = import ./mb { inherit config; }; }
     { krebs = import ./nin    { inherit config; }; }
     { krebs = import ./external/palo.nix { inherit config; }; }
     { krebs = import ./tv     { inherit config; }; }
diff --git a/krebs/3modules/external/default.nix b/krebs/3modules/external/default.nix
index 247dae69c..821859f3c 100644
--- a/krebs/3modules/external/default.nix
+++ b/krebs/3modules/external/default.nix
@@ -359,8 +359,8 @@ in {
       nets = rec {
         retiolum = {
           addrs = [
-            config.krebs.hosts.donna.nets.retiolum.ip4.addr
-            config.krebs.hosts.donna.nets.retiolum.ip6.addr
+            config.krebs.hosts.amy.nets.retiolum.ip4.addr
+            config.krebs.hosts.amy.nets.retiolum.ip6.addr
           ];
           ip4.addr = "10.243.29.181";
           aliases = [ "amy.r" ];
@@ -387,8 +387,8 @@ in {
       nets = rec {
         retiolum = {
           addrs = [
-            config.krebs.hosts.donna.nets.retiolum.ip4.addr
-            config.krebs.hosts.donna.nets.retiolum.ip6.addr
+            config.krebs.hosts.clara.nets.retiolum.ip4.addr
+            config.krebs.hosts.clara.nets.retiolum.ip6.addr
           ];
           ip4.addr = "10.243.29.182";
           aliases = [ "clara.r" ];
diff --git a/krebs/3modules/iana-etc.nix b/krebs/3modules/iana-etc.nix
index f6d47f27e..e8037128d 100644
--- a/krebs/3modules/iana-etc.nix
+++ b/krebs/3modules/iana-etc.nix
@@ -23,32 +23,20 @@ with import <stockholm/lib>;
   };
 
   config.environment.etc = mkIf (config.krebs.iana-etc.services != {})  {
-    services.source = mkForce (pkgs.runCommand "krebs-iana-etc" {} ''
-      exec < ${pkgs.iana_etc}/etc/services
-      exec > $out
-      awk -F '[ /]+' '
-        BEGIN {
-          port=0
-        }
-        ${concatMapStringsSep "\n" (entry: ''
-          $2 == ${entry.port} {
-            port=$2
-            next
-          }
-          port == ${entry.port} {
-            ${concatMapStringsSep "\n"
-              (proto: let
-                s = "${entry.${proto}.name} ${entry.port}/${proto}";
-              in
-                "print ${toJSON s}")
-              (filter (proto: entry.${proto} != null) ["tcp" "udp"])}
-            port=0
-          }
-        '') (attrValues config.krebs.iana-etc.services)}
-        {
-          print $0
-        }
-      '
+    services.source = mkForce (pkgs.runCommand "krebs-iana-etc" {} /* sh */ ''
+      {
+        ${concatMapStringsSep "\n" (entry: /* sh */ ''
+          ${concatMapStringsSep "\n"
+            (proto: let
+              line = "${entry.${proto}.name} ${entry.port}/${proto}";
+            in /* sh */ ''
+              echo ${shell.escape line}
+            '')
+            (filter (proto: entry.${proto} != null) ["tcp" "udp"])}
+          '') (attrValues config.krebs.iana-etc.services)}
+        cat ${pkgs.iana_etc}/etc/services
+      } |
+      sort -b -k 2,2 -u > $out
     '');
   };
 
diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix
index 30c7b085f..00847071a 100644
--- a/krebs/3modules/lass/default.nix
+++ b/krebs/3modules/lass/default.nix
@@ -111,44 +111,6 @@ in {
       ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsANFdMi825qWQXQbWLYuNZ6/fARt3lnh1KStQHQQMD";
       syncthing.id = "QITFKYQ-VEPIPL2-AZIXHMD-BBT62ML-YHSB35A-BSUIBXS-QYMPFHW-M7XN2QU";
     };
-    archprism = {
-      cores = 1;
-      nets = rec {
-        internet = {
-          ip4.addr = "46.4.114.247";
-          aliases = [
-            "archprism.i"
-          ];
-          ssh.port = 45621;
-        };
-        retiolum = {
-          via = internet;
-          ip4.addr = "10.243.0.123";
-          aliases = [
-            "archprism.r"
-          ];
-          tinc.pubkey = ''
-            -----BEGIN PUBLIC KEY-----
-            MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA6dK0jsPSb7kWMGjfyWbG
-            wQYYt8vi5pY/1/Ohk0iy84+mfb1SCJdm5IOC4WXgHtmfd468OluUpU5etAu13D3n
-            f0iDeCuohH0uTjP+EojnKrAXYTiTRpySqXjVmhaWwFyMAACFdzKFb9cgMoByrP0U
-            5qruBcupK8Zwxt+Pe8IadRpPuOmz/bMYS7r+NKwybttoIX+YVm4myNzqdtMT77+H
-            BYR2mzW99T5YI54YZoCe0+XiIEQsosd6IL/9dP0+6vku6nHLD4qb81Q9AgaT+hte
-            s/ivHL+Fe2GULEQUi8aoEfXrPwnGFVY+QYxLw2G9A0Gfe9KnYBXDn99HXUGcFu2l
-            x7duN6mnT3WNC6VReh9m5+rPMnih/3l82W0tH1lBWUtdKcxx6yhkyUFgKOvkm4UP
-            gf1+EIpxf+bM7jlWylKGc+bD+dTMFV+tzHE6qHlcnzdZQrhYd0zjOXGnm4Kl1ec5
-            GSlpmqTcjgR+42l6frAENo3fndqYw1WkDtswImDz3Wjuco7BiOULHTJvQN+Ao1DI
-            l2MQDOWJoN4eYIE4XPqLSvdOSavHQB2WGv+dFDDpWOxnDLNi19aubtynIfpGJXxV
-            L8s9kUTG00Hdv08BG06hGt0+2Sy1PTVniDcTftHKmEOPS6Y5rJzQih7JdakSUQCc
-            6j/HwgWTf85Io/tbVMTNtkECAwEAAQ==
-            -----END PUBLIC KEY-----
-          '';
-        };
-      };
-      ssh.privkey.path = <secrets/ssh.id_ed25519>;
-      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsANFdMi825qWQXQbWLYuNZ6/fARt3lnh1KStQHQQMD";
-    };
-
     uriel = {
       monitoring = false;
       cores = 1;
diff --git a/krebs/3modules/mb/default.nix b/krebs/3modules/mb/default.nix
deleted file mode 100644
index 31e01c4ab..000000000
--- a/krebs/3modules/mb/default.nix
+++ /dev/null
@@ -1,151 +0,0 @@
-with import <stockholm/lib>;
-{ config, ... }: let
-
-  hostDefaults = hostName: host: flip recursiveUpdate host {
-    ci = true;
-    owner = config.krebs.users.mb;
-  };
-
-in {
-  hosts = mapAttrs hostDefaults {
-    orange = {
-      nets = {
-        retiolum = {
-          ip4.addr = "10.243.42.23";
-          aliases = [
-            "orange.r"
-            "or4ng3.r"
-            "0r4n93.r"
-          ];
-          tinc.pubkey = ''
-            -----BEGIN PUBLIC KEY-----
-            MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAr7P0CkmC5HWnTdgGFzmA
-            zQuJzHSkSjcGgSkIt0pvqU6xi8P/d4eJlmeXeGTpH62JfM1xhEMpxMVd/4NOON2u
-            IlWnfu5bB763145IJwE0HmZziWjQXWRPAZMqYdQ5f2Pvmxv1yr3uBNzr8UlV6BjD
-            FXn8sCvikXttYzts9szlz5+pkY09qfiz48+DMzRBNO6JzXYQ9kPyS+TIXlGpN4Jp
-            C1TRF38eF2DTEZ58Yx8Z99dGrXVuqlSe77fehTQGxCckTpaZ0HS3XfZNa/cas8JY
-            /0RzH2n2AndnPirISDZ7r4ZIFuKAaivqaEkM8v7llI77URVB9ZJb/IqCrBzueAbt
-            V/5ts2HpfBAUhw0RoiH8ql+IQZsuSOpRUC2gUN8460V4SQkVtDcsVTENiD+NM5Mg
-            ImBv041CsW/rSJOilT2r/rWDN8RFnz/RrAQn+L31KXr81kg1TOLxO0ybs/eMJM3r
-            RnHFZPiiKdqPlA60g0AnzKXPR2JTszHIgHHoRUW16I1WJeuAJNjg0JDQ0JM7pZ27
-            JEaCc7uR12TPiuExKaNEaxKZVY1J0hzxOzF2MFIbAMVz/3K2ycvvuLxKojqIAXxA
-            D+UtcOfJ62k2WnLXOEIZqFU0J2bvhxYUZOFS55wIn1UJF7hemD/LUFHBiWnuhwHk
-            TAEl8M851t+Zp3hZeJzgx2kCAwEAAQ==
-            -----END PUBLIC KEY-----
-          '';
-        };
-      };
-    };
-    rofl = {
-      nets = {
-        retiolum = {
-          ip4.addr = "10.243.42.43";
-          aliases = [
-            "rofl.r"
-          ];
-          tinc.pubkey = ''
-            -----BEGIN PUBLIC KEY-----
-            MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnysdVVwxkmSroNUleYZm
-            xdaIB9EdZYCo2xj3WyhsD2lWMpj51FzSH6Y052Vy1V1TCuIXIwjidpmMohBvflG8
-            txKCaBGQOZbVqRgzyCDXsNisbr05ayYuHcRrXTpn5ask4HN0Vtx2uJOn8YmOxA0D
-            VhyEnf8xWu+vi8dwDqRVR17QnPBYqgenzIBmAuRngvNqg6WZg+E9X2e1Dco/PMzb
-            VW0AgC2+zFCl4+G7dEW7uhsI6IJLy4LsJuEN4TlvWAf7tfdFEnBzTfODW8quGdts
-            1Yzah4svPNNt9F1ZhOR/1bDsfVoOjI76BgB0G+ZZPQAGV1zxgn8DXSKi/tJTLNu1
-            vj/n9sUJfXMYQdTAOkABghCyEDFUspPKCffQqUXUcJbLKY9fNssGGBeanMsobUQC
-            Ch9z7kIJ52JDcP/D58z9Yf62P5ENqXzeVPCcodIOey1EizOu/FH3jVo52we1M5sp
-            1iM4hMc3ZINUBI9AA1nLWWlB3lBnErAXrhmMMHjcO4nO7/M0YU+EalkDB5eIhqiH
-            QJx7VnOE2UZYU9Y0vVNSWfYocU12aABK98T7lr5Tde4dI1J81sk2MUZcbNHger3f
-            NxpvNzOBpeC5xvq/ENCRR7MDf/59xWW5P5N7PbGprLQAi8cfdSoIEhSPz17Taq1f
-            3aAAePgBsZvRQozxXZfqp58CAwEAAQ==
-            -----END PUBLIC KEY-----
-          '';
-        };
-      };
-    };
-    p1nk = {
-      nets = {
-        retiolum = {
-          ip4.addr = "10.243.42.42";
-          aliases = [
-            "p1nk.r"
-          ];
-          tinc.pubkey = ''
-            -----BEGIN PUBLIC KEY-----
-            MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA5YVML71oW3iJrzZKuX48
-            AKrGitO5zNvsAHOI8BVsGfZTyxAAZgG4OaDX45kr27K39NcBU43LdDD0I1yjNvGe
-            zAoL5MIiCPD/QR1kAvLmgpMUSqOVvrk+uoGLVt6dOGvxlOiG1AAaN0gA8Q0B/jZV
-            4tZlBpZ7MX9xeK10wqVT56msN69P3EzKQn1uoVRrBxEnNvI1iqmmkgMLcrFVJFBQ
-            888Uuw9Hx5MO7ES/ATe8mt0zReUGvn91jYVVsPpmAopWnjCol271gflY0RomFXKy
-            XaIuvbeF+3otF0+MNqJfm4IsAKJjvl92pjVX0f0eBCSPCYR7D1EtgQrqflLkZKZ8
-            jBGDlgpsFWt/Omz1BYcuGZU/djM4+SNxr4YRYMi3lMix3s2PmHvm304I7eEEBlC9
-            qy1jq/sLaf8mHJrF6Htl7W5WS/Famkwv/VreI92iHrhsmIDiX7OIbXzYDCxT/PQa
-            6uCm/3jIbcHG/ZHZ12H6thkafK0Aoe009+p1n+5Y7V2oNvYe3KzZTnCN5t6z1QHZ
-            V5iypsd6lNDzlodjleTgGK8FmHGRPRdq1wb3eOLE8mWZj7ygDT50FwaC8FzAcHgC
-            bLN/zlHvCbYmk9IJhktO3B6wtMrZl60+XCpb5rTulM94RirifFYsnTIDJApI11yb
-            3AYi5dQXHjab/lvj6917xa0CAwEAAQ==
-            -----END PUBLIC KEY-----
-          '';
-        };
-      };
-    };
-    gr33n = {
-      nets = {
-        retiolum = {
-          ip4.addr = "10.243.42.123";
-          aliases = [
-            "gr33n.r"
-          ];
-          tinc.pubkey = ''
-            -----BEGIN PUBLIC KEY-----
-            MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvcqecLfk8TlGFF7JJpv8
-            kjLFNgoNfu9FYRMNG4GSxWL5w+49n6b+GC5ciOC+RJ+N56jfB9JYE0MtbuOmkY5M
-            JUphuvgOCNhTbBJsjnmG9n02evpxZn9HWypNC3oQCvY2K7vHpIxGKR5PyTVKPO0P
-            OOYKAbCLD9F2bmuLaBQ/dFXFQxfu3tjvJI9mYDWBpSkh1mYeMZLw2xxnRZLs0bEO
-            ZWdzxCh9UM/mgb4WYuNED9+sz7MSsaMPAqquarFCguUxhjp6rElGFcNWjXaxA5zt
-            JGS6VompUViVSHjSaQ5/3VRKoIQjr4NOFYQqLpmB5S2OpiggV6I9OpB7QUGlvcYd
-            I3j+1AeK11HuEyPqSwxjNCCrI12bSIo3685BPHbl+AMhWGhzrCkAGcOCbAefreXQ
-            5v4SaKUIDlCYhN+vyNdlu2jvqQlxfJrPAfBt+jJBK6gMcAEKc7P/Oj4B9Fsl331X
-            s0kWH5G9t6OhO/Of8/kb2/P+YEbM6zi1QQdZAOr6Cg0y4cMt9zxLWknaM4yEDAXH
-            oSM33PTv9DOvBjfxRXqOHqOHRq5ayqZdIFgfLUlPTdbWRkhNzjG8f7k7p32m20A6
-            Kal+OF//I2c9E9vKFzyepyTcnwi3B8+cFJ74+XYaNApdwHSb1BU/+c3O9RJExZGV
-            jtTSbSJHU5esECtAuXy1XH8CAwEAAQ==
-            -----END PUBLIC KEY-----
-          '';
-        };
-      };
-    };
-    sunsh1n3 = {
-      ci = false;
-      nets = {
-        retiolum = {
-          ip4.addr = "10.243.42.142";
-          aliases = [
-            "sunsh1n3.r"
-          ];
-          tinc.pubkey = ''
-            -----BEGIN PUBLIC KEY-----
-            MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAo2VCqp6mUbyo3n+1XpKf
-            QavpgRYQyv9wAZzYSYHjxThuLmNb/wERPbWJFMZfAGuku0blKWJISSgFWd9YL7dU
-            pZQZxfqo/9xnS/r0xIKrKSsBiTZt7JZmTQzj1ri11TIO0S1QPjIP5HsxlZZAw0nz
-            idEDlKmgWs74FPdezlXqvJyEUKDqL/ZQBtdhZZIDMkSJnCdBzXxKwv+uLVE46ZBf
-            4HrtQjcj+dyVMogMIoseAgf5lS6V3pyCM7/NHZFxrIxoIAxSsUoB59i2EbK6aUK5
-            yuiWHI6ZHToxN2K/0SX96hzxcwrUmdk49tTHBY0Zhn2ku6NjQPU3LuxgIwrSaSJD
-            /KWh6XkqR7EsCVN0AIsLvFelI2ckSyNyAlnYbMAHDt7GwHlNp4Lsy+x4ZQ6m0xTY
-            Z+/jt6sfoMiulPcwWEpqNCCf5A65lF77DldQhH3qYrdQ756n/kOqSfQtPCnVNYXy
-            LlN5rKCOgxKxxtKkwMUif2OM9RPHpM7wS09Rvek6zpL9ymhU5THF7UylLKxKGjYj
-            6dTooyRVQRJdrwIYLrJIy0MfGyYiGAJxf/C0KOOZnJPCW2b51+bo5Zh+BhKZYN8H
-            C2DEGc8+4h5hX1TAaUfTpfVm3mMTh8H2m9N8Pdl5ji+A0m0IwHDLQyaoskcxSjvU
-            9IxYLfkSD6AJqasnHlz0L08CAwEAAQ==
-            -----END PUBLIC KEY-----
-          '';
-        };
-      };
-    };
-  };
-  users = {
-    mb = {
-      mail = "mb0@codemonkey.cc";
-      pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDCHAdKGHP/De/GLEsPo5RBfbaiiitMw4Y/akOekJbImswT6Np2lzqno/WBJcfVs3D39wgPKNld4P/QZc5IwxC26q/PnBFu93KES0GqnlAqUNE63IOJ8UzNdyEqWggnRiLrBU+ZgyFZvmqp6NoSO4YEGEK4RZRMJM/GcAuQMj/nGjx2AHwPGZCkIRgz8/ctBOzX1/knZd3cOnNowH1wlqUKX6UcEzJdAVDQijHF1wl0Ri8tJKq9u8s/fw+1PSOpOHaeF1BALsXSKgeJDqUCTnZW5mAVUWJ86LvvyfCP4In9lhhLisbDm2cD96QaVvJyV6HfmegdSxZ1Phh+9Qz+3WhDJRedBTSKWfK/9j7VWSb+z/KV37q72W25ZfFMSay58LmCqn3v5fGt9qj4nlPw0By4baGLiGlA7xyvkJfdt8ZVPps5d2g6UprTbSA79lYN4qtWKq2Z9t317xch7Lix6EunQcoTkJ6QXEbDrAIk3zvkWr/CtpwEhNcSdWvQsua42dkD2oOI2F2IgFyYgOx9Iba2yj8A0TD2iqfYVhsJIYuk12QfeaR7ovQ6DhHlUxyQzeF6h0Y+I4AN6Sq/Mmj/cxfQoIaAEybUQMX+7KjFceIszT3JbGlz7DCxi7DMmNYuc7LELMRG3jNAOk+fW8u42Bhgc44tzvAondojerUGqCbUDw== mb0@codemonkey.cc";
-    };
-  };
-}
diff --git a/krebs/3modules/realwallpaper.nix b/krebs/3modules/realwallpaper.nix
index a0c00c20d..c09bb008d 100644
--- a/krebs/3modules/realwallpaper.nix
+++ b/krebs/3modules/realwallpaper.nix
@@ -29,7 +29,7 @@ let
 
     cloudmap = mkOption {
       type = types.str;
-      default = "http://xplanetclouds.com/free/local/clouds_2048.jpg";
+      default = "http://home.megapass.co.kr/~holywatr/cloud_data/clouds_2048.jpg";
     };
 
     marker = mkOption {
diff --git a/krebs/5pkgs/simple/newsbot-js/default.nix b/krebs/5pkgs/simple/newsbot-js/default.nix
index 055e6b476..0ac66f433 100644
--- a/krebs/5pkgs/simple/newsbot-js/default.nix
+++ b/krebs/5pkgs/simple/newsbot-js/default.nix
@@ -1,11 +1,11 @@
-{ stdenv, makeWrapper, lib, buildEnv, fetchgit, nodejs-12_x, pkgs, icu }:
+{ stdenv, makeWrapper, lib, buildEnv, fetchgit, nodejs, pkgs, icu }:
 
 with lib;
 
 let
   nodeEnv = import <nixpkgs/pkgs/development/node-packages/node-env.nix> {
     inherit (pkgs) stdenv python2 utillinux runCommand writeTextFile;
-    nodejs = nodejs-12_x;
+    nodejs = nodejs;
     libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
   };
 
@@ -36,7 +36,7 @@ in stdenv.mkDerivation {
   ];
 
   buildInputs = [
-    nodejs-12_x
+    nodejs
     makeWrapper
   ];
 
@@ -45,7 +45,7 @@ in stdenv.mkDerivation {
 
     cp newsbot.js $out/
     cat > $out/newsbot << EOF
-      ${nodejs-12_x}/bin/node $out/newsbot