Merge remote-tracking branch 'tv/master'

12 files changed, 274 insertions, 23 deletions

diff --git a/krebs/2configs/reaktor2.nix b/krebs/2configs/reaktor2.nix
index b52125ae8..f4fa2ca7c 100644
--- a/krebs/2configs/reaktor2.nix
+++ b/krebs/2configs/reaktor2.nix
@@ -70,7 +70,8 @@ let
             filename = pkgs.writeDash "bier-balance" ''
               ${pkgs.hledger}/bin/hledger -f $state_file bal -N -O csv \
                 | ${pkgs.coreutils}/bin/tail +2 \
-                | ${pkgs.miller}/bin/mlr --icsv --opprint cat
+                | ${pkgs.miller}/bin/mlr --icsv --opprint cat \
+                | ${pkgs.gnused}/bin/sed 's/^/the_/'
             '';
           };
         }
diff --git a/krebs/3modules/external/default.nix b/krebs/3modules/external/default.nix
index 1720811d9..66f9620c7 100644
--- a/krebs/3modules/external/default.nix
+++ b/krebs/3modules/external/default.nix
@@ -229,6 +229,105 @@ in {
         };
       };
     };
+    rose = {
+      owner = config.krebs.users.Mic92;
+      nets = rec {
+        internet = {
+          ip4.addr = "129.215.165.52";
+          aliases = [ "rose.i" ];
+        };
+        retiolum = {
+          via = internet;
+          addrs = [
+            config.krebs.hosts.rose.nets.retiolum.ip4.addr
+            config.krebs.hosts.rose.nets.retiolum.ip6.addr
+          ];
+          ip4.addr = "10.243.29.178";
+          aliases = [ "rose.r" ];
+          tinc.pubkey = ''
+            -----BEGIN RSA PUBLIC KEY-----
+            MIICCgKCAgEA0h88uEcgVFhggGh3xqHySt8T+oDdoSN8ve4ZPmMzrGCD4dnlWcUO
+            6uMiwE7XG667wvjB0J2RbCJ8n8/r6eQgp6sRfPzSQL/Mc74J+py+sOVOjjjL5wJX
+            btrYmASO3GKUSMhGmM0IiwHMIPrmUViaREDrweF3bUwK45d/ocqpBkc+nF27kksd
+            DMYjHMWRIkKuQaj592zo/kY1pAJ/yAvDPess0x1CLL6uDNbjTr2S/L7JHdzZs9Xq
+            1+SGdVtqD0sWgSBKA0PC/Mi+Divd4PC1SoSL7wZRWD0Y2DNgj3+xUc7hAWRCw2Gs
+            5wofK+qiwnyYAmeNYcyQfDLosKZF9hOM8U3UbxptkPLsOK3cfZoGoLQCuOryVDBe
+            6GfJkJ49WfuSSNWs3WPWL6/6zmVPeGR0TvoMt02VQ3cKTmeIkWyTIzSVoC7wYv5D
+            Dl8Xt3aFr9UFI2GxenesViyuDLi8cy2fOsM3r+gowXQtgEKoXc9W2vyPwIIlcWUJ
+            QrKVsyNlkKKL0YjsnGazaEvqdiE30/Iq7f7VBnXnWXRLnZhr85HbTdDQnpT4GcEv
+            W3jpl1y5zShr5Hz90QoYcUTsxg9uk/+yqKpwUySZ6Gh4q0bo5k7nkM9i8mCMfNGZ
+            0UU94QmwS9RoV4Mt4pSLYRcCs0mVeEjLuIfTFHkXc6LCjBWMn8ICfeMCAwEAAQ==
+            -----END RSA PUBLIC KEY-----
+          '';
+        };
+      };
+    };
+    martha = {
+      owner = config.krebs.users.Mic92;
+      nets = rec {
+        internet = {
+          ip4.addr = "129.215.165.53";
+          aliases = [ "martha.i" ];
+        };
+        retiolum = {
+          via = internet;
+          addrs = [
+            config.krebs.hosts.martha.nets.retiolum.ip4.addr
+            config.krebs.hosts.martha.nets.retiolum.ip6.addr
+          ];
+          ip4.addr = "10.243.29.179";
+          aliases = [ "martha.r" ];
+          tinc.pubkey = ''
+            -----BEGIN RSA PUBLIC KEY-----
+            MIICCgKCAgEA3lR3Wup2yd9SYs9n9a7lq/jXxlKdwjgp9gPEirLn3/XCFM7NpLIp
+            LRm3Wdplv0NWim4zI3AsdGmUBrV3y0Ugj48Td4RpXlOiFjS8NHnvRbamCZF7m/pJ
+            3T/QpQx98+QEKXb3gZ5aDGgcHLRbUYUBuwFOxAKaikuDe2qJxqXqOmA7RXZDkEqe
+            FrQE/H1/+8HqJ1vhgZKi3Vu7zLRB1EV8nggWFjQKR8o0AeViLwM3OxFtGyKTaXuK
+            WAQrvSdKQDpQwqAPogyeftGesOfW7z0xrelkux10p42YM9epYvZDFRG97/nupw/S
+            iYGiTTFDBDTzpyT3zl1uwhmQ3re/nJXf5e4fgnZEcsweU8ysHtDhbimqrm9impVn
+            XdKnnuNa9F8VlyHCT2pVC9+WDKDNtA2M8f+8lG8/hoJ7hhp5HhBZ3ncROyQqOg4F
+            e6YtaFidi+fYXjQkdUXHv5FCkqFJnoxZdI2vwqU2DumltG/o+qsksI2WSsLsuMVs
+            sa4KUq0+5OsmCJnIAKWV2YwbLVf1tJMjPGA0jQECrHPL6SKobRefqav6MPuTbytC
+            4frtEIGbfdKqQ6nNTvTpCrAo+WAm3NE3khTYqGe4LqX/JMoGtWXp/Ex9IdG+sflM
+            mESMjuHp9vPY4aZGPtYPP93Cxv3q7gm+EfIGebajISpaG28J+XjiNNsCAwEAAQ==
+            -----END RSA PUBLIC KEY-----
+          '';
+        };
+      };
+    };
+    donna = {
+      owner = config.krebs.users.Mic92;
+      nets = rec {
+        internet = {
+          ip4.addr = "129.215.165.54";
+          aliases = [ "donna.i" ];
+        };
+        retiolum = {
+          via = internet;
+          addrs = [
+            config.krebs.hosts.donna.nets.retiolum.ip4.addr
+            config.krebs.hosts.donna.nets.retiolum.ip6.addr
+          ];
+          ip4.addr = "10.243.29.180";
+          aliases = [ "donna.r" ];
+          tinc.pubkey = ''
+            -----BEGIN RSA PUBLIC KEY-----
+            MIICCgKCAgEAnv5zVPwjHk5Q72D3tv2rlQkp7SOsZD7Wvz8l1yI/mWkxoriJ9MVa
+            x8RziSB3KF8sF1lRWIKmuynkgLI3w0X/YFs/fAvtayxk6Qf8DOl23Vd8Is0h/i3I
+            0fCmCEIHhHboKsREW6NxY7w5WAI2+SFNmGef1P7vzrAv7iLyPbo9nQ8wlrAmc+PJ
+            Ao3BOf4U7kP778fhsPA4dlGtF2v9CBhygeGVI/DQR8jcvzeiPd2Dr0k/JvrVMYtf
+            wJW4xUwZkIpws/yfI8b4VJOFl2X/Yw9712Z8Jvga0rR32OG4YbnggvuCMum1g94k
+            YwMjaSckv1XTalvPQuf1Od96XzwL2hjPFpEK3Tdl4AitMnArgj9HNzhcRL+eGonf
+            U24zk52OToHnoP3palNpodi7DziIBeXIaIMl7VMXku2ymbOUJsI6zeew+uZahJkv
+            QIWjxveQ8N40BoTc8Yg6pea1AId3l4f3brtwJbQOVbb3bVQ5VcrxM9Q/TBvyADYR
+            Knwszxw3uBw5Za1FMbwCPwd8/y/Ar19qGCx25xK0QnsyqZZT/cHsbBOTzh6BBWwI
+            IzbYu49VO/B1rktYzZ2l2ENQy6OILXWbvFjC8Pt8f1ZZQ4A21PyNA1AdyJ/rbVj7
+            awm3OnnvKSvMCXWnwHPFHjksb3qMx96Aep1cw3ZBx0sQQ41UWBoOsi8CAwEAAQ==
+            -----END RSA PUBLIC KEY-----
+          '';
+        };
+      };
+    };
     inspector = {
       owner = config.krebs.users.Mic92;
       nets = rec {
@@ -536,6 +635,13 @@ in {
         };
       };
     };
+    domsen-backup = {
+      owner = config.krebs.users.domsen;
+      ci = false;
+      external = true;
+      syncthing.id = "22NLFY5-QMRM3BH-76QIBYI-OPMKVGM-DU4FNZI-3KN2POF-V4WIC6M-2SFFUAC";
+      nets = {};
+    };
   };
   users = {
     ciko = {
@@ -581,5 +687,7 @@ in {
     filly = {
     };
     pie_ = {};
+    domsen = {
+    };
   };
 }
diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix
index f4c8f5c6a..217edfdd1 100644
--- a/krebs/3modules/lass/default.nix
+++ b/krebs/3modules/lass/default.nix
@@ -393,6 +393,55 @@ in {
       ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJzb9BPFClubs6wSOi/ivqPFVPlowXwAxBS0jHaB29hX";
       syncthing.id = "PCDXICO-GMGWKSB-V6CYF3I-LQMZSGV-B7YBJXA-DVO7KXN-TFCSQXW-XY6WNQD";
     };
+    xerxes = {
+      cores = 2;
+      nets = rec {
+        retiolum = {
+          ip4.addr = "10.243.1.3";
+          ip6.addr = r6 "3";
+          aliases = [
+            "xerxes.r"
+          ];
+          tinc.pubkey = ''
+            -----BEGIN RSA PUBLIC KEY-----
+            MIIECgKCBAEArqEaK+m7WZe/9/Vbc+qx2TjkkRJ9lDgDMr1dvj98xb8/EveUME6U
+            MZyAqNjLuKq3CKzJLo02ZmdFs4CT1Hj28p5IC0wLUWn53hrqdy8cCJDvIiKIv+Jk
+            gItsxJyMnRtsdDbB6IFJ08D5ReGdAFJT5lqpN0DZuNC6UQRxzUK5fwKYVVzVX2+W
+            /EZzEPe5XbE69V/Op2XJ2G6byg9KjOzNJyJxyjwVco7OXn1OBNp94NXoFrUO7kxb
+            mTNnh3D+iB4c3qv8woLhmb+Uh/9MbXS14QrSf85ou4kfUjb5gdhjIlzz+jfA/6XO
+            X4t86uv8L5IzrhSGb0TmhrIh5HhUmSKT4RdHJom0LB7EASMR2ZY9AqIG11XmXuhj
+            +2b5INBZSj8Cotv5aoRXiPSaOd7bw7lklYe4ZxAU+avXot9K3/4XVLmi6Wa6Okim
+            hz+MEYjW5gXY+YSUWXOR4o24jTmDjQJpdL83eKwLVAtbrE7TcVszHX6zfMoQZ5M9
+            3EtOkDMxhC+WfkL+DLQAURhgcPTZoaj0cAlvpb0TELZESwTBI09jh/IBMXHBZwI4
+            H1gOD5YENpf0yUbLjVu4p82Qly10y58XFnUmYay0EnEgdPOOVViovGEqTiAHMmm5
+            JixtwJDz7a6Prb+owIg27/eE1/E6hpfXpU8U83qDYGkIJazLnufy32MTFE4T9fI4
+            hS8icFcNlsobZp+1pB3YK4GV5BnvMwOIVXVlP8yMCRTDRWZ4oYmAZ5apD7OXyNwe
+            SUP2mCNNlQCqyjRsxj5S1lZQRy1sLQztU5Sff4xYNK+5aPgJACmvSi3uaJAxBloo
+            4xCCYzxhaBlvwVISJXZTq76VSPybeQ+pmSZFMleNnWOstvevLFeOoH2Is0Ioi1Fe
+            vnu5r0D0VYsb746wyRooiEuOAjBmni8X/je6Vwr1gb/WZfZ23EwYpGyakJdxLNv3
+            Li+LD9vUfOR80WL608sUU45tAx1RAy6QcH/YDtdClbOdK53+cQVTsYnCvDW8uGlO
+            scQWgk+od3qvo6yCPO7pRlEd3nedcPSGh/KjBHao6eP+bsVERp733Vb9qrEVwmxv
+            jlZ1m12V63wHVu9uMAGi9MhK+2Q/l7uLTj03OYpi4NYKL2Bu01VXfoxuauuZLdIJ
+            Z3ZV+qUcjzZI0PBlGxubq6CqVFoSB7nhHUbcdPQ66WUnwoKq0cKmE7VOlJQvJ07u
+            /Wsl8BIsxODVt0rTzEAx0hTd5mJCX7sCawRt+NF+1DZizl9ouebNMkNlsEAg4Ps0
+            bQerZLcOmpYjGa5+lWDwJIMXVIcxwTmQR86stlP/KQm0vdOvH2ZUWTXcYvCYlHkQ
+            sgVnnA2wt+7UpZnEBHy04ry+jYaSsPdYgwIDAQAB
+            -----END RSA PUBLIC KEY-----
+          '';
+        };
+        wiregrill = {
+          ip6.addr = w6 "3";
+          aliases = [
+            "xerxes.w"
+          ];
+          wireguard.pubkey = "UTm8B8YUVvBGqwwxAUMVFsVQFQGQ6jbcXAavZ8LxYT8=";
+        };
+      };
+      secure = true;
+      ssh.privkey.path = <secrets/ssh.id_ed25519>;
+      ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE5HyLyaIvVH0qHIQ4ciKhDiElhSqsK+uXcA6lTvL+5n";
+      syncthing.id = "EA76ZHP-DF2I3CJ-NNTFEUH-YGPQK5S-T7FQ6JA-BNQQUNC-GF2YL46-CKOZCQM";
+    };
     red = {
       monitoring = false;
       cores = 1;
@@ -626,7 +675,7 @@ in {
     };
     lass-xerxes = {
       mail = "lass@xerxes.r";
-      pubkey = builtins.readFile ./ssh/xerxes.rsa;
+      pubkey = builtins.readFile ./ssh/xerxes.ed25519;
     };
     lass-daedalus = {
       mail = "lass@daedalus.r";
diff --git a/krebs/3modules/lass/ssh/xerxes.ed25519 b/krebs/3modules/lass/ssh/xerxes.ed25519
new file mode 100644
index 000000000..87a40ca2a
--- /dev/null
+++ b/krebs/3modules/lass/ssh/xerxes.ed25519
@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGwCq56DGqj/kz8d8ax0xIl29jV9f3tUtDgtnCnS1b4q lass@xerxes
diff --git a/krebs/3modules/lass/ssh/xerxes.rsa b/krebs/3modules/lass/ssh/xerxes.rsa
deleted file mode 100644
index 2b5da7b25..000000000
--- a/krebs/3modules/lass/ssh/xerxes.rsa
+++ /dev/null
@@ -1 +0,0 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAEZgGjBN0aFs6GxNwMjCvlddbN6+vb6LZuWiWWe+wbAynaGuGbae0TXCLp0/eMNy7fH8poDjpdW9M4mKbBFKOqyG8WJLCPFoQw761tjKl1hccJn0hFSkQAEGKxtfzHlAl/Mz+59yvqNg7/WNSivv41hE7btYltzRy238VQDYFv2eLM7acyxrgGo7tWOtkbpfELj5cM8Qw1j3TF9bGV5pK6IOEtaHbmalS8Iiz77syAu+6E/y6zKBTtGMHI15l6RNJ/Y7A1LM/WwuNL+9dJMYWJFVHy3/4dpaxiioHREiSawUbz9wNHknCrT6vaPCIVVcujhz9Oee1C5UiYUyyfJrFYdlzaTg7FuLNIt2hKMY6NYx1D8/Pwpq1JOsaEfK/K5ytCgaJb115mRevcaUA5s7KYNWHmmZvy08JzCgSM6ZPRtfkQIcha77wVq6DugJ+KgBz+oADQRKiaMrumOMldd0B3q4Oxb71gDTE1XLAbWJnd/0Up1H5GAtZZUUrMUslZiU/23R6SOkyEMLWQTx/KgkWcz8DZLtib5o03uZpfJDVqp2CR+sjmy4x9aa+lSaOzuZP0KRyg+mOKl0o3zL7TNAzrzSCORVBg7nOh+0SPJkDxVRkc6dVY1L3ZOfdm2P/19fhWEr5ECgVrmYYKnDPwWY1iWJlZsiEc3Mj7KB1m44ov0FJg2hiNnydImqcXTCoszp515MBmeHnpqJsqEZuWS8dAnaEiOwZaSKIO1E7lQ7CoP86+eD4yAwLq6fb2tgjHT69LgDMaIha4hMfrO2o4UDVw9OZMfnPtyatI4pxplaQDoQM1p0dej0rZ7uxL1tfoKAyT0UCdtjhxfnNs0x1gOQbML4eGbqyKuyF82eOQRgKRDqH/tParoE4SRBVi7o3s0kILRmXA3ng3n1uhEiGwPTH8JsQ9huM+XOhH8+CzQeg4yb/jCrhsDzvLaW654+ouq9G+kjwqmO4vLNs5eZxfae84rppbS2MJqK1x8rkJixvKBKEfvYJOuDNV+hXyMbToaq8qtGy7cCSq4+UDio3DsSHY0Tpt9e+yEzoOOqFQLQyq6uHv/+u9MY+VADoa4N64U3S2SXul9tE3g6hOAY0F5BYMbxQSuj59kzwghlAmbsyWN2FCmWdsfCQkkZX7wCTj20DtZB/GdVSGNgHGAoU5JZrXKca3A2Yc9hzbYjyNYr0NmQ9NUbkbaOkcYJRIUXtS2OBOHP+FoUkkqL3ieKXR07l5xJbWLzbyVUxN9Zii4Baj5xnDO/RLZPDvTUxbER/0d1orMZztL2EKmfSn4j4uhWqpi04Rg9sWH+WVLAq22EKhAuqcFEOUimjcyZWYKxcAq5Z51NJNBQB7euz55eCJUZkBUYEpNuYr0UDlmBxKB2r6ZWDeNXT7eLxBdwDHCHSqXV7qOG1vMhHtjbbxmQMnkQ4InhO9TdpaN3tj67nGmc6hhgYO4b7NvyL1/pvDPrHrR/3GzkDkwqvt3uESdVdqAJSCk6gFh9V1aGs= lass@xerxes
diff --git a/krebs/3modules/mb/default.nix b/krebs/3modules/mb/default.nix
index e77811f08..31e01c4ab 100644
--- a/krebs/3modules/mb/default.nix
+++ b/krebs/3modules/mb/default.nix
@@ -36,6 +36,32 @@ in {
         };
       };
     };
+    rofl = {
+      nets = {
+        retiolum = {
+          ip4.addr = "10.243.42.43";
+          aliases = [
+            "rofl.r"
+          ];
+          tinc.pubkey = ''
+            -----BEGIN PUBLIC KEY-----
+            MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnysdVVwxkmSroNUleYZm
+            xdaIB9EdZYCo2xj3WyhsD2lWMpj51FzSH6Y052Vy1V1TCuIXIwjidpmMohBvflG8
+            txKCaBGQOZbVqRgzyCDXsNisbr05ayYuHcRrXTpn5ask4HN0Vtx2uJOn8YmOxA0D
+            VhyEnf8xWu+vi8dwDqRVR17QnPBYqgenzIBmAuRngvNqg6WZg+E9X2e1Dco/PMzb
+            VW0AgC2+zFCl4+G7dEW7uhsI6IJLy4LsJuEN4TlvWAf7tfdFEnBzTfODW8quGdts
+            1Yzah4svPNNt9F1ZhOR/1bDsfVoOjI76BgB0G+ZZPQAGV1zxgn8DXSKi/tJTLNu1
+            vj/n9sUJfXMYQdTAOkABghCyEDFUspPKCffQqUXUcJbLKY9fNssGGBeanMsobUQC
+            Ch9z7kIJ52JDcP/D58z9Yf62P5ENqXzeVPCcodIOey1EizOu/FH3jVo52we1M5sp
+            1iM4hMc3ZINUBI9AA1nLWWlB3lBnErAXrhmMMHjcO4nO7/M0YU+EalkDB5eIhqiH
+            QJx7VnOE2UZYU9Y0vVNSWfYocU12aABK98T7lr5Tde4dI1J81sk2MUZcbNHger3f
+            NxpvNzOBpeC5xvq/ENCRR7MDf/59xWW5P5N7PbGprLQAi8cfdSoIEhSPz17Taq1f
+            3aAAePgBsZvRQozxXZfqp58CAwEAAQ==
+            -----END PUBLIC KEY-----
+          '';
+        };
+      };
+    };
     p1nk = {
       nets = {
         retiolum = {
diff --git a/krebs/3modules/syncthing.nix b/krebs/3modules/syncthing.nix
index 939c8fddf..799ed7eda 100644
--- a/krebs/3modules/syncthing.nix
+++ b/krebs/3modules/syncthing.nix
@@ -176,6 +176,7 @@ in
   config = mkIf kcfg.enable {
 
     systemd.services.syncthing = mkIf (kcfg.cert != null || kcfg.key != null) {
+      serviceConfig.PermissionsStartOnly = mkDefault true;
       preStart = ''
         ${optionalString (kcfg.cert != null) ''
           cp ${toString kcfg.cert} ${scfg.configDir}/cert.pem
diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix
index 0cec1a2d3..43535b08f 100644
--- a/krebs/3modules/urlwatch.nix
+++ b/krebs/3modules/urlwatch.nix
@@ -16,7 +16,6 @@ let
 
   api = {
     enable = mkEnableOption "krebs.urlwatch";
-
     dataDir = mkOption {
       type = types.str;
       default = "/var/lib/urlwatch";
@@ -54,6 +53,18 @@ let
         The format is described in systemd.time(7), CALENDAR EVENTS.
       '';
     };
+    sendmail.enable = mkEnableOption "krebs.urlwatch.sendmail" // {
+      default = true;
+    };
+    telegram = {
+      enable = mkEnableOption "krebs.urlwatch.telegram";
+      botToken = mkOption {
+        type = types.str;
+      };
+      chatId = mkOption {
+        type = types.listOf types.str;
+      };
+    };
     urls = mkOption {
       type = with types; listOf (either str subtypes.job);
       default = [];
@@ -64,10 +75,7 @@ let
       ];
       apply = map (x: getAttr (typeOf x) {
         set = x;
-        string = {
-          url = x;
-          filter = null;
-        };
+        string.url = x;
       });
     };
     verbose = mkOption {
@@ -85,7 +93,7 @@ let
 
   hooksFile = cfg.hooksFile;
 
-  configFile = pkgs.writeText "urlwatch.yaml" (toJSON {
+  configFile = pkgs.writeJSON "urlwatch.yaml" {
     display = {
       error = true;
       new = true;
@@ -110,13 +118,18 @@ let
         color = true;
         enabled = true;
       };
+      ${if cfg.telegram.enable then "telegram" else null} = {
+        enabled = cfg.telegram.enable;
+        bot_token = cfg.telegram.botToken;
+        chat_id = cfg.telegram.chatId;
+      };
       text = {
         details = true;
         footer = true;
         line_length = 75;
       };
     };
-  });
+  };
 
   imp = {
     systemd.timers.urlwatch = {
@@ -158,19 +171,21 @@ let
               --urls=${shell.escape urlsFile} \
             > changes || :
 
-          if test -s changes; then
-            {
-              echo Date: $(date -R)
-              echo From: ${shell.escape cfg.from}
-              echo Subject: $(
-                sed -n 's/^\(CHANGED\|ERROR\|NEW\): //p' changes \
-                  | tr '\n' ' '
-              )
-              echo To: ${shell.escape cfg.mailto}
-              echo
-              cat changes
-            } | /run/wrappers/bin/sendmail -t
-          fi
+          ${optionalString cfg.sendmail.enable /* sh */ ''
+            if test -s changes; then
+              {
+                echo Date: $(date -R)
+                echo From: ${shell.escape cfg.from}
+                echo Subject: $(
+                  sed -n 's/^\(CHANGED\|ERROR\|NEW\): //p' changes \
+                    | tr '\n' ' '
+                )
+                echo To: ${shell.escape cfg.mailto}
+                echo
+                cat changes
+              } | /run/wrappers/bin/sendmail -t
+            fi
+          ''}
         '';
       };
     };
@@ -192,8 +207,13 @@ let
         type = types.str;
       };
       filter = mkOption {
+        default = null;
         type = with types; nullOr str; # TODO nullOr subtypes.filter
       };
+      ignore_cached = mkOption {
+        default = null;
+        type = with types; nullOr bool;
+      };
     };
   };
 in out
diff --git a/krebs/5pkgs/haskell/blessings.nix b/krebs/5pkgs/haskell/blessings.nix
index 7abebba56..55f2d17d0 100644
--- a/krebs/5pkgs/haskell/blessings.nix
+++ b/krebs/5pkgs/haskell/blessings.nix
@@ -14,6 +14,10 @@ with import <stockholm/lib>;
       version = "2.2.0";
       sha256 = "1pb56dgf3jj2kq3cbbppwzyg3ccgqy9xara62hkjwyxzdx20clk1";
     };
+    "19.09" = {
+      version = "2.2.0";
+      sha256 = "1pb56dgf3jj2kq3cbbppwzyg3ccgqy9xara62hkjwyxzdx20clk1";
+    };
   }.${versions.majorMinor version};
 
 in mkDerivation {
diff --git a/krebs/5pkgs/haskell/email-header.nix b/krebs/5pkgs/haskell/email-header.nix
index fe4bb839b..8b7165860 100644
--- a/krebs/5pkgs/haskell/email-header.nix
+++ b/krebs/5pkgs/haskell/email-header.nix
@@ -20,6 +20,11 @@ with import <stockholm/lib>;
       rev = "refs/tags/v${cfg.version}";
       sha256 = "11xjivpj495r2ss9aqljnpzzycb57cm4sr7yzmf939rzwsd3ib0x";
     };
+    "19.09" = {
+      version = "0.4.1-tv1";
+      rev = "refs/tags/v${cfg.version}";
+      sha256 = "11xjivpj495r2ss9aqljnpzzycb57cm4sr7yzmf939rzwsd3ib0x";
+    };
   }.${versions.majorMinor version};
 
 in mkDerivation {
diff --git a/krebs/5pkgs/override/default.nix b/krebs/5pkgs/override/default.nix
new file mode 100644
index 000000000..704831823
--- /dev/null
+++ b/krebs/5pkgs/override/default.nix
@@ -0,0 +1,12 @@
+with import <stockholm/lib>;
+self: super: {
+
+  exim = super.exim.overrideAttrs (old: rec {
+    name = warnOldVersion old.name "exim-4.92.2";
+    src = self.fetchurl {
+      url = "https://ftp.exim.org/pub/exim/exim4/${name}.tar.xz";
+      sha256 = "0m56jsh2fzvwj4rdpcc3pkd5vsi40cjrpzalis7l1zq33m4axmq1";
+    };
+  });
+
+}
diff --git a/krebs/5pkgs/simple/nix-prefetch-github.nix b/krebs/5pkgs/simple/nix-prefetch-github.nix
new file mode 100644
index 000000000..14096c33f
--- /dev/null
+++ b/krebs/5pkgs/simple/nix-prefetch-github.nix
@@ -0,0 +1,25 @@
+{ curl, jq, nix, writeDashBin }:
+
+writeDashBin "nix-prefetch-github" ''
+  # usage: nix-prefetch-github OWNER REPO [REF]
+  set -efu
+
+  owner=$1
+  repo=$2
+  ref=''${3-master}
+
+  info_url=https://api.github.com/repos/$owner/$repo/commits/$ref
+  info=$(${curl}/bin/curl -fsS "$info_url")
+  rev=$(printf %s "$info" | ${jq}/bin/jq -r .sha)
+
+  name=$owner-$repo-$ref
+  url=https://github.com/$owner/$repo/tarball/$rev
+  sha256=$(${nix}/bin/nix-prefetch-url --name "$name" --unpack "$url")
+
+  export owner repo rev sha256
+  ${jq}/bin/jq -n '
+    env | {
+      owner, repo, rev, sha256
+    }
+  '
+''