Merge remote-tracking branch 'ni/master'

7 files changed, 174 insertions, 14 deletions

diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix
index 2e336de21..113f6e65d 100644
--- a/krebs/3modules/urlwatch.nix
+++ b/krebs/3modules/urlwatch.nix
@@ -71,7 +71,7 @@ let
       description = "URL to watch.";
       example = [
         https://nixos.org/channels/nixos-unstable/git-revision
-        { url = http://localhost ; filter = "grep:important.*stuff"; }
+        { url = http://localhost ; filter = [ (grep "important.*stuff") ]; }
       ];
       apply = map (x: getAttr (typeOf x) {
         set = x;
@@ -177,12 +177,15 @@ let
                 echo Date: $(date -R)
                 echo From: ${shell.escape cfg.from}
                 echo Subject: $(
-                  sed -n 's/^\(CHANGED\|ERROR\|NEW\): //p' changes \
-                    | tr '\n' ' '
+                  sed -nr 's/^(CHANGED|ERROR|NEW): //p' changes |
+                  sed '1!s/^  //'
                 )
                 echo To: ${shell.escape cfg.mailto}
+                echo Mime-Version: 1.0
+                echo Content-Type: text/plain\; charset=UTF-8
+                echo Content-Transfer-Encoding: base64
                 echo
-                cat changes
+                base64 changes
               } | /run/wrappers/bin/sendmail -t
             fi
           ''}
@@ -211,7 +214,9 @@ let
       };
       filter = mkOption {
         default = null;
-        type = with types; nullOr str; # TODO nullOr subtypes.filter
+        type =
+          with types;
+          nullOr (either str (listOf (pkgs.formats.json {}).type));
       };
       ignore_cached = mkOption {
         default = null;
diff --git a/krebs/5pkgs/simple/nixos-format-error.nix b/krebs/5pkgs/simple/nixos-format-error.nix
new file mode 100644
index 000000000..a28f7245f
--- /dev/null
+++ b/krebs/5pkgs/simple/nixos-format-error.nix
@@ -0,0 +1,107 @@
+{ pkgs }:
+
+pkgs.writeGawkBin "nixos-format-error" ''
+  # usage: nixos-rebuild ... 2>&1 | nixos-format-error
+
+  function out() {
+    print
+    next
+  }
+
+  BEGIN {
+    IDLE = 0
+    ACTIVE = 1
+    PASSIVE = 2
+    ERROR = 3
+
+    start_state = IDLE
+
+    state = start_state
+  }
+
+  END {
+    if (trace_count)
+      for (i = trace_count - 1; i >= 0; i--)
+        print trace[i]
+  }
+
+  state == PASSIVE {
+    out()
+  }
+
+  state == IDLE {
+    if ($0 ~ /^building the system configuration\.\.\.
?$/) {
+      state = ACTIVE
+    }
+    out()
+  }
+
+  state == ACTIVE {
+    if ($1 ~ /(\[[0-9;]+m)?error:(\[[0-9;]m)?/) {
+      state = ERROR
+      sub(/^/,"\x1b[31;1m"); sub(/$/,"\x1b[m")
+      trace[trace_count++] = $0
+
+      "stty -F /dev/tty size" |& getline
+      COLUMNS = gensub(/.* ([0-9]+)$/, "\\1", "1")
+
+      next
+    }
+    if ($0 ~ /^these [0-9]+ derivations will be built:/) {
+      state = PASSIVE
+    }
+    if ($0 == "activating the configuration...") {
+      state = PASSIVE
+    }
+    out()
+  }
+
+  state == ERROR {
+    sub(/
$/, "")
+    gsub(/\[[0-9;]*m/, "")
+
+    if ($0 ~ /^\s*at /) {
+      location = gensub(/^\s*at (.*):$/,"\\1","1")
+      content = ""
+      lnumcol = gensub(/^.*:([0-9]+:[0-9]+)$/,"\\1","1",location)
+      lnum = gensub(/:.*/,"","1",lnumcol)
+      col = gensub(/.*:/,"","1",lnumcol)
+      next
+    }
+
+    if ($1 == lnum "|") {
+      content = gensub(/^\s*[0-9]+\|(.*)/,"\\1","1")
+
+      location = sprintf("%50s", location)
+
+      preview_size = COLUMNS - length(location " ")
+
+      prefix = gensub(/^\s*/,"","1",substr(content, 1, col))
+      infix = gensub(/^([0-9a-zA-Z]+|.).*$/, "\\1", "1", substr(content, col + 1))
+      suffix = substr(content, col + length(infix) + 1)
+
+      if (length(prefix infix suffix) > preview_size) {
+        n = (preview_size - length(infix)) / 2 - length(" ")
+        prefix = substr(prefix, length(prefix) - n + 1)
+        if (prefix != "") { prefix = "…" prefix }
+        suffix = substr(suffix, 1, n)
+        if (suffix != "") { suffix = suffix "…" }
+      }
+
+      preview = \
+        "\x1b[38;5;244m" prefix "\x1b[m" \
+        "\x1b[38;5;230m" infix "\x1b[m" \
+        "\x1b[38;5;244m" suffix "\x1b[m"
+
+      trace[trace_count++] = location " " preview
+      next
+    }
+
+    if ($0 == "") next
+    if ($0 ~ /^\s*… (from|while)/) next
+    if ($0 ~ /^\s*([0-9]*)\|/) next
+
+    trace[trace_count++] = $0
+    next
+  }
+''
diff --git a/submodules/nix-writers b/submodules/nix-writers
-Subproject 0c8de150426476b5287cf2787bbd85263691a80
+Subproject d1424777b5f2e12cbd80efd1b55335dcba7c32b
diff --git a/tv/1systems/bu/config.nix b/tv/1systems/bu/config.nix
index c7f7da24d..a3959cd84 100644
--- a/tv/1systems/bu/config.nix
+++ b/tv/1systems/bu/config.nix
@@ -13,8 +13,6 @@ with import ./lib;
 
   krebs.build.host = config.krebs.hosts.bu;
 
-  networking.hostId = lib.mkDefault "00000000";
-
   networking.wireless.enable = true;
   networking.useDHCP = false;
   networking.interfaces.enp0s25.useDHCP = true;
diff --git a/tv/2configs/default.nix b/tv/2configs/default.nix
index d1384845a..53b11c620 100644
--- a/tv/2configs/default.nix
+++ b/tv/2configs/default.nix
@@ -6,6 +6,7 @@ with import ./lib;
 
   krebs.build.user = config.krebs.users.tv;
 
+  networking.hostId = mkDefault (hashToLength 8 config.networking.hostName);
   networking.hostName = config.krebs.build.host.name;
 
   imports = [
diff --git a/tv/2configs/urlwatch.nix b/tv/2configs/urlwatch.nix
index 7ba364ff3..f5260ee05 100644
--- a/tv/2configs/urlwatch.nix
+++ b/tv/2configs/urlwatch.nix
@@ -2,12 +2,16 @@ with import ./lib;
 { config, pkgs, ... }: let
   exec = filename: args: url: {
     inherit url;
-    filter = "system:${
-      concatMapStringsSep " " shell.escape ([filename] ++ toList args)
-    }";
+    filter = singleton {
+      system =
+        concatMapStringsSep " " shell.escape ([filename] ++ toList args);
+    };
   };
   json = json' ["."];
   json' = exec "${pkgs.jq}/bin/jq";
+  urigrep' = exec (pkgs.writeDash "urigrep" ''
+    ${pkgs.urix}/bin/urix | ${pkgs.gnugrep}/bin/grep -E "$1"
+  '');
   xml = xml' ["--format" "-"];
   xml' = exec "${pkgs.libxml2}/bin/xmllint";
 in {
@@ -68,22 +72,30 @@ in {
       https://raw.githubusercontent.com/NixOS/nixpkgs/master/nixos/modules/services/x11/xserver.nix
 
       https://www.rabbitmq.com/changelog.html
+
+      (urigrep' ["software-resources"] https://semiconductor.samsung.com/consumer-storage/support/tools/)
     ];
     hooksFile = toFile "hooks.py" ''
       import subprocess
       import urlwatch
 
-      class CaseFilter(urlwatch.filters.FilterBase):
+      class SystemFilter(urlwatch.filters.FilterBase):
           """Filter for piping data through an external process"""
 
           __kind__ = 'system'
 
+          __supported_subfilters__ = {
+              'command': 'shell command line to tranform data',
+          }
+
+          __default_subfilter__ = 'command'
+
           def filter(self, data, subfilter=None):
-              if subfilter is None:
-                  raise ValueError('The system filter needs a command')
+              if 'command' not in subfilter:
+                  raise ValueError('{} filter needs a command'.format(self.__kind__))
 
               proc = subprocess.Popen(
-                  subfilter,
+                  subfilter['command'],
                   shell=True,
                   stdin=subprocess.PIPE,
                   stdout=subprocess.PIPE,
diff --git a/tv/2configs/wiregrill.nix b/tv/2configs/wiregrill.nix
new file mode 100644
index 000000000..d28a1ec29
--- /dev/null
+++ b/tv/2configs/wiregrill.nix
@@ -0,0 +1,37 @@
+with import ./lib;
+{ config, pkgs, ... }: let
+  cfg = {
+    enable = cfg.net != null;
+    net = config.krebs.build.host.nets.wiregrill or null;
+  };
+  toCidrNotation = ip: "${ip.addr}/${toString ip.prefixLength}";
+in
+  mkIf cfg.enable {
+    networking.wireguard.interfaces.wiregrill = {
+      ips =
+        optional (cfg.net.ip4 != null) cfg.net.ip4.addr ++
+        optional (cfg.net.ip6 != null) cfg.net.ip6.addr;
+      listenPort = 51820;
+      privateKeyFile = (toString <secrets>) + "/wiregrill.key";
+      allowedIPsAsRoutes = true;
+      peers = mapAttrsToList
+        (_: host: {
+          allowedIPs = host.nets.wiregrill.wireguard.subnets;
+          endpoint =
+            mkIf (host.nets.wiregrill.via != null) (host.nets.wiregrill.via.ip4.addr + ":${toString host.nets.wiregrill.wireguard.port}");
+          persistentKeepalive = mkIf (host.nets.wiregrill.via != null) 61;
+          publicKey =
+            replaceStrings ["\n"] [""] host.nets.wiregrill.wireguard.pubkey;
+        })
+        (filterAttrs (_: h: hasAttr "wiregrill" h.nets) config.krebs.hosts);
+    };
+    systemd.network.networks.wiregrill = {
+      matchConfig.Name = "wiregrill";
+      address =
+        optional (!isNull cfg.net.ip4) (toCidrNotation cfg.net.ip4) ++
+        optional (!isNull cfg.net.ip6) (toCidrNotation cfg.net.ip6);
+    };
+    tv.iptables.extra.filter.INPUT = [
+      "-p udp --dport ${toString cfg.net.wireguard.port} -j ACCEPT"
+    ];
+  }