k shack/alerting: prepare irc-alerts for binaergewitter

4 files changed, 271 insertions, 21 deletions

diff --git a/krebs/2configs/shack/prometheus/alert-rules.nix b/krebs/2configs/shack/prometheus/alert-rules.nix
index 5ba49ede..4cefdc3e 100644
--- a/krebs/2configs/shack/prometheus/alert-rules.nix
+++ b/krebs/2configs/shack/prometheus/alert-rules.nix
@@ -1,6 +1,6 @@
 { lib,... }:
 let
-  disk_free_threshold = "10"; # at least this much free disk percentage
+  disk_free_threshold = "5"; # at least this much free disk percentage
 in {
   services.prometheus.rules = [(builtins.toJSON
     {
@@ -8,22 +8,6 @@ in {
         { name = "shack-env";
           rules = [
             {
-              alert = "Wolf RootPartitionFull";
-              for = "30m";
-              expr = ''(node_filesystem_avail_bytes{alias="wolf.shack",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="wolf.shack",mountpoint="/"} < ${disk_free_threshold}'';
-              labels.severity = "warning";
-              annotations.summary = "{{ $labels.alias }} root disk full";
-              annotations.url = "http://grafana.shack/d/hb7fSE0Zz/shack-system-dashboard?orgId=1&var-job=node&var-hostname=All&var-node=wolf.shack:9100&var-device=All&var-maxmount=%2F&var-show_hostname=wolf";
-              annotations.description = ''The root disk of {{ $labels.alias }} has {{ $value | printf "%.2f" }}% free disk space (Threshold at ${disk_free_threshold}%). CI for deploying new configuration will seize working. Log in to the system and try to clean up the obsolete files on the machine. There are a couple of things you can do:
-1. `nix-collect-garbage -d`
-2. clean up the shack share folder in `/home/share`
-3. check `du -hs /var/ | sort -h`.
-4. run `docker system prune`
-5. `find /var/lib/containers/news/var/lib/htgen-go/items -mtime +7 -delete;` to clean up the link shortener data
-5. If you are really desperate run `du -hs / | sort -h` and go through the folders recursively until you've found something to delete
-6. as a last resort the root disk can be expanded via `lvresize -L +10G /dev/pool/root && btrfs filesystem resize max /` '';
-            }
-            {
               alert = "Puyak RootPartitionFull";
               for = "30m";
               expr = ''(node_filesystem_avail_bytes{alias="puyak.shack",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="puyak.shack",mountpoint="/"} < ${disk_free_threshold}'';
@@ -32,9 +16,8 @@ in {
               annotations.url = "http://grafana.shack/d/hb7fSE0Zz/shack-system-dashboard?orgId=1&var-job=node&var-hostname=All&var-node=wolf.shack:9100&var-device=All&var-maxmount=%2F&var-show_hostname=puyak";
               annotations.description = ''The root disk of {{ $labels.alias }} has {{ $value | printf "%.2f" }}% free disk space (Threshold at ${disk_free_threshold}%).Prometheus will not be able to create new alerts and CI for deploying new configuration will also seize working. Log in to the system and run `nix-collect-garbage -d` and if this does not help you can check `du -hs /var/ | sort -h`, run `docker system prune` or if you are really desperate run `du -hs / | sort -h` and go through the folders recursively until you've found something to delete'';
             }
-            # wolf.shack is not worth supervising anymore
             {
-              alert = "HostDown";
+              alert = "Infra01 down";
               expr = ''up{alias="infra01.shack"} == 0'';
               for = "5m";
               labels.severity = "page";
diff --git a/krebs/2configs/shack/prometheus/irc-alerts.py b/krebs/2configs/shack/prometheus/irc-alerts.py
new file mode 100644
index 00000000..005a2013
--- /dev/null
+++ b/krebs/2configs/shack/prometheus/irc-alerts.py
@@ -0,0 +1,207 @@
+import base64
+import cgi
+import json
+import os
+import re
+import socket
+import ssl
+import sys
+from http.server import BaseHTTPRequestHandler
+from typing import List, Optional, Tuple
+from urllib.parse import urlparse
+
+DEBUG = os.environ.get("DEBUG") is not None
+
+
+def _irc_send(
+    server: str,
+    nick: str,
+    channel: str,
+    sasl_password: Optional[str] = None,
+    server_password: Optional[str] = None,
+    tls: bool = True,
+    port: int = 6697,
+    messages: List[str] = [],
+) -> None:
+    if not messages:
+        return
+
+    sock = socket.socket()
+    if tls:
+        sock = ssl.wrap_socket(
+            sock, cert_reqs=ssl.CERT_NONE, ssl_version=ssl.PROTOCOL_TLSv1_2
+        )
+
+    def _send(command: str) -> int:
+        if DEBUG:
+            print(command)
+        return sock.send((f"{command}\r\n").encode())
+
+    def _pong(ping: str):
+        if ping.startswith("PING"):
+            sock.send(ping.replace("PING", "PONG").encode("ascii"))
+
+    recv_file = sock.makefile(mode="r")
+
+    print(f"connect {server}:{port}")
+    sock.connect((server, port))
+    if server_password:
+        _send(f"PASS {server_password}")
+    _send(f"USER {nick} 0 * :{nick}")
+    _send(f"NICK {nick}")
+    for line in recv_file.readline():
+        if re.match(r"^:[^ ]* (MODE|221|376|422) ", line):
+            break
+        else:
+            _pong(line)
+
+    if sasl_password:
+        _send("CAP REQ :sasl")
+        _send("AUTHENTICATE PLAIN")
+        auth = base64.encodebytes(f"{nick}\0{nick}\0{sasl_password}".encode("utf-8"))
+        _send(f"AUTHENTICATE {auth.decode('ascii')}")
+        _send("CAP END")
+    _send(f"JOIN :{channel}")
+
+    for m in messages:
+        _send(f"PRIVMSG {channel} :{m}")
+
+    _send("INFO")
+    for line in recv_file:
+        if DEBUG:
+            print(line, end="")
+        # Assume INFO reply means we are done
+        if "End of /INFO" in line:
+            break
+        else:
+            _pong(line)
+
+    sock.send(b"QUIT")
+    print("disconnect")
+    sock.close()
+
+
+def irc_send(
+    url: str, notifications: List[str], password: Optional[str] = None
+) -> None:
+    parsed = urlparse(f"{url}")
+    username = parsed.username or "prometheus"
+    server = parsed.hostname or "chat.freenode.net"
+    if parsed.fragment != "":
+        channel = f"#{parsed.fragment}"
+    else:
+        channel = "#krebs-announce"
+    port = parsed.port or 6697
+    if not password:
+        password = parsed.password
+    if len(notifications) == 0:
+        return
+    _irc_send(
+        server=server,
+        nick=username,
+        sasl_password=password,
+        channel=channel,
+        port=port,
+        messages=notifications,
+        tls=parsed.scheme == "irc+tls",
+    )
+
+
+class PrometheusWebHook(BaseHTTPRequestHandler):
+    def __init__(
+        self,
+        irc_url: str,
+        conn: socket.socket,
+        addr: Tuple[str, int],
+        password: Optional[str] = None,
+    ) -> None:
+        self.irc_url = irc_url
+        self.password = password
+        self.rfile = conn.makefile("rb")
+        self.wfile = conn.makefile("wb")
+        self.client_address = addr
+        self.handle()
+
+    # for testing
+    def do_GET(self) -> None:
+        if DEBUG:
+            print("GET: Request Received")
+        self.send_response(200)
+        self.send_header("Content-type", "text/plain")
+        self.end_headers()
+        self.wfile.write(b"ok")
+
+    def do_POST(self) -> None:
+        if DEBUG:
+            print("POST: Request Received")
+        content_type, _ = cgi.parse_header(self.headers.get("content-type"))
+
+        # refuse to receive non-json content
+        if content_type != "application/json":
+            if DEBUG:
+                print(f"POST: wrong content type {content_type}")
+            self.send_response(400)
+            self.end_headers()
+            return
+
+        length = int(self.headers.get("content-length"))
+        payload = json.loads(self.rfile.read(length))
+        messages = []
+        for alert in payload["alerts"]:
+            description = alert["annotations"]["description"]
+            messages.append(f"{alert['status']}: {description}")
+        irc_send(self.irc_url, messages, password=self.password)
+
+        self.do_GET()
+
+
+def systemd_socket_response() -> None:
+    irc_url = os.environ.get("IRC_URL", None)
+    if irc_url is None:
+        print(
+            "IRC_URL environment variable not set: i.e. IRC_URL=irc+tls://mic92-prometheus@chat.freenode.net/#krebs-announce",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    password = None
+    irc_password_file = os.environ.get("IRC_PASSWORD_FILE", None)
+    if irc_password_file:
+        with open(irc_password_file) as f:
+            password = f.read()
+
+    msgs = sys.argv[1:]
+
+    if msgs != []:
+        irc_send(irc_url, msgs, password=password)
+        return
+
+    nfds = os.environ.get("LISTEN_FDS", None)
+    if nfds is None:
+        print(
+            "LISTEN_FDS not set. Run me with systemd(TM) socket activation?",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+    fds = range(3, 3 + int(nfds))
+
+    for fd in fds:
+        sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
+        sock.settimeout(0)
+
+        try:
+            while True:
+                PrometheusWebHook(irc_url, *sock.accept(), password=password)
+        except BlockingIOError:
+            # no more connections
+            pass
+
+
+if __name__ == "__main__":
+    if DEBUG:
+        print("Starting in DEBUG mode")
+    if len(sys.argv) == 3:
+        print(f"{sys.argv[1]} {sys.argv[2]}")
+        irc_send(sys.argv[1], [sys.argv[2]])
+    else:
+        systemd_socket_response()
diff --git a/krebs/2configs/shack/prometheus/irc-hooks.nix b/krebs/2configs/shack/prometheus/irc-hooks.nix
new file mode 100644
index 00000000..07bb2423
--- /dev/null
+++ b/krebs/2configs/shack/prometheus/irc-hooks.nix
@@ -0,0 +1,59 @@
+{ config
+, lib
+, pkgs
+, ...
+}:
+let
+  irc-alerts = pkgs.writers.writePython3 "irc-alerts" { 
+    flakeIgnore = [ "E501" ];
+  } (builtins.readFile ./irc-alerts.py);
+  endpoints = {
+    binaergewitter = {
+      url = "irc+tls://puyak-alerts@irc.libera.chat:6697/#binaergewitter-alerts";
+      port = 9223;
+    };
+  };
+in
+{
+  systemd.sockets =
+    lib.mapAttrs'
+      (name: opts:
+        lib.nameValuePair "irc-alerts-${name}" {
+          description = "Receive http hook and send irc message for ${name}";
+          wantedBy = [ "sockets.target" ];
+          listenStreams = [ "[::]:${builtins.toString opts.port}" ];
+        }) endpoints;
+
+  systemd.services =
+    lib.mapAttrs'
+      (name: opts:
+        let
+          serviceName = "irc-alerts-${name}";
+          hasPassword = opts.passwordFile or null != null;
+        in
+        lib.nameValuePair serviceName {
+          description = "Receive http hook and send irc message for ${name}";
+          requires = [ "irc-alerts-${name}.socket" ];
+          serviceConfig =
+            {
+              Environment =
+                [
+                  "IRC_URL=${opts.url}"
+                  "DEBUG=y"
+                ]
+                ++ lib.optional hasPassword "IRC_PASSWORD_FILE=/run/${serviceName}/password";
+              DynamicUser = true;
+              User = serviceName;
+              ExecStart = irc-alerts;
+            }
+            // lib.optionalAttrs hasPassword {
+              PermissionsStartOnly = true;
+              ExecStartPre =
+                "${pkgs.coreutils}/bin/install -m400 "
+                + "-o ${serviceName} -g ${serviceName} "
+                + "${config.sops.secrets.prometheus-irc-password.path} "
+                + "/run/${serviceName}/password";
+              RuntimeDirectory = serviceName;
+            };
+        }) endpoints;
+}
diff --git a/krebs/2configs/shack/prometheus/server.nix b/krebs/2configs/shack/prometheus/server.nix
index 9e4b4d1a..7a553202 100644
--- a/krebs/2configs/shack/prometheus/server.nix
+++ b/krebs/2configs/shack/prometheus/server.nix
@@ -3,6 +3,7 @@
 {
   imports = [
     ./alert-rules.nix
+    ./irc-hooks.nix
   ];
   networking = {
     firewall.allowedTCPPorts = [
@@ -129,11 +130,11 @@
             "group_wait" = "30s";
             "group_interval" = "2m";
             "repeat_interval" = "4h";
-            "receiver" = "team-admins";
+            "receiver" = "shack-admins";
           };
           "receivers" = [
             {
-              "name" = "team-admins";
+              "name" = "shack-admins";
               "email_configs" = [ ];
               "webhook_configs" = [
                 {