Merge remote-tracking branch 'lass/master'

5 files changed, 48 insertions, 165 deletions

diff --git a/lass/2configs/monitoring/client.nix b/lass/2configs/monitoring/client.nix
deleted file mode 100644
index b8c245215..000000000
--- a/lass/2configs/monitoring/client.nix
+++ /dev/null
@@ -1,26 +0,0 @@
-{pkgs, config, ...}:
-with import <stockholm/lib>;
-{
-  services.telegraf = {
-    enable = true;
-
-    extraConfig = {
-      agent.interval = "1s";
-      outputs = {
-        influxdb = {
-          urls = ["http://prism:8086"];
-          database = "telegraf_db";
-          user_agent = "telegraf";
-        };
-      };
-      inputs = {
-        cpu = {
-          percpu = false;
-          totalcpu = true;
-        };
-        mem = {};
-        net = {};
-      };
-    };
-  };
-}
diff --git a/lass/2configs/monitoring/monit-alarms.nix b/lass/2configs/monitoring/monit-alarms.nix
deleted file mode 100644
index 2cfc292e5..000000000
--- a/lass/2configs/monitoring/monit-alarms.nix
+++ /dev/null
@@ -1,44 +0,0 @@
-{pkgs, config, ...}:
-with import <stockholm/lib>;
-let
-  echoToIrc = msg:
-    pkgs.writeDash "echo_irc" ''
-      set -euf
-      export LOGNAME=prism-alarm
-      ${pkgs.irc-announce}/bin/irc-announce \
-        irc.r 6667 ${config.networking.hostName}-alarm \#noise "${msg}" >/dev/null
-    '';
-
-in {
-  krebs.monit = {
-    enable = true;
-    http.enable = true;
-    alarms = {
-      nirwanabluete = {
-        test = "${pkgs.curl}/bin/curl -sf 'https://nirwanabluete.de/'";
-        alarm = echoToIrc "test nirwanabluete failed";
-      };
-      ubik = {
-        test = "${pkgs.curl}/bin/curl -sf 'https://ubikmedia.de'";
-        alarm = echoToIrc "test ubik failed";
-      };
-      cac-panel = {
-        test = "${pkgs.curl}/bin/curl -sf 'https://panel.cloudatcost.com/login.php'";
-        alarm = echoToIrc "test cac-panel failed";
-      };
-      radio = {
-        test = pkgs.writeBash "check_stream" ''
-          ${pkgs.curl}/bin/curl -sif http://lassul.us:8000/radio.ogg \
-          | ${pkgs.gawk}/bin/awk '/^\r$/{exit}{print $0}' \
-          | ${pkgs.gnugrep}/bin/grep -q "200 OK" || exit "''${PIPESTATUS[0]}"
-        '';
-        alarm = echoToIrc "test radio failed";
-      };
-    };
-  };
-
-  krebs.iptables.tables.filter.INPUT.rules = [
-    { predicate = "-p tcp -i retiolum --dport 9093"; target = "ACCEPT"; }
-  ];
-}
-
diff --git a/lass/2configs/monitoring/node-exporter.nix b/lass/2configs/monitoring/node-exporter.nix
index 8c27e90d4..561e3a25c 100644
--- a/lass/2configs/monitoring/node-exporter.nix
+++ b/lass/2configs/monitoring/node-exporter.nix
@@ -1,7 +1,9 @@
 { config, lib, pkgs, ... }:
 {
-  networking.firewall.allowedTCPPorts = [ 9100 ];
-
+  krebs.iptables.tables.filter.INPUT.rules = [
+    { predicate = "-i retiolum -p tcp --dport 9100 -s ${config.krebs.hosts.prism.nets.retiolum.ip4.addr}"; target = "ACCEPT"; v6 = false; }
+    { predicate = "-i retiolum -p tcp --dport 9100 -s ${config.krebs.hosts.prism.nets.retiolum.ip6.addr}"; target = "ACCEPT"; v4 = false; }
+  ];
   services.prometheus.exporters = {
     node = {
       enable = true;
diff --git a/lass/2configs/monitoring/prometheus-server.nix b/lass/2configs/monitoring/prometheus-server.nix
index d56d7e552..e16d421a0 100644
--- a/lass/2configs/monitoring/prometheus-server.nix
+++ b/lass/2configs/monitoring/prometheus-server.nix
@@ -9,6 +9,12 @@
   #  useDHCP = true;
   #};
 
+  krebs.iptables.tables.filter.INPUT.rules = [
+    { predicate = "-i retiolum -p tcp --dport 3000"; target = "ACCEPT"; }
+    { predicate = "-i retiolum -p tcp --dport 9090"; target = "ACCEPT"; }
+    { predicate = "-i retiolum -p tcp --dport 9093"; target = "ACCEPT"; }
+  ];
+
   services = {
     prometheus = {
       enable = true;
@@ -124,11 +130,10 @@
           static_configs = [
             {
               targets = [
-                "localhost:9100"
-              ];
-              labels = {
-                alias = "prometheus.example.com";
-              };
+              ] ++ map (host: "${host}:9100") (lib.attrNames (lib.filterAttrs (_: host: host.owner.name == "lass" && host.monitoring) config.krebs.hosts));
+              #labels = {
+              #  alias = "prometheus.example.com";
+              #};
             }
           ];
         }
@@ -159,7 +164,7 @@
               ];
               "webhook_configs" = [
                 {
-                  "url" = "https://example.com/prometheus-alerts";
+                  "url" = "http://127.0.0.1:14813/prometheus-alerts";
                   "send_resolved" = true;
                 }
               ];
@@ -176,4 +181,37 @@
       security = import <secrets/grafana_security.nix>; # { AdminUser = ""; adminPassword = ""}
     };
   };
+  services.logstash = {
+    enable = true;
+    inputConfig = ''
+      http {
+        port => 14813
+        host => "127.0.0.1"
+      }
+    '';
+    filterConfig = ''
+      if ([alerts]) {
+        ruby {
+          code => '
+            lines = []
+            event["alerts"].each {|p|
+              lines << "#{p["labels"]["instance"]}#{p["annotations"]["summary"]} #{p["status"]}"
+            }
+            event["output"] = lines.join("\n")
+          '
+        }
+      }
+    '';
+    outputConfig = ''
+      file { path => "/tmp/logs.json" codec => "json_lines" }
+      irc {
+        channels => [ "#noise" ]
+        host => "irc.r"
+        nick => "alarm"
+        codec => "json_lines"
+        format => "%{output}"
+      }
+    '';
+    #plugins = [ ];
+  };
 }
diff --git a/lass/2configs/monitoring/server.nix b/lass/2configs/monitoring/server.nix
deleted file mode 100644
index adaecde2c..000000000
--- a/lass/2configs/monitoring/server.nix
+++ /dev/null
@@ -1,87 +0,0 @@
-{pkgs, config, ...}:
-with import <stockholm/lib>;
-{
-  services.influxdb.enable = true;
-
-  services.influxdb.extraConfig = {
-    meta.hostname = config.krebs.build.host.name;
-    # meta.logging-enabled = true;
-    http.bind-address = ":8086";
-    admin.bind-address = ":8083";
-    http.log-enabled = false;
-    monitoring = {
-      enabled = false;
-      # write-interval = "24h";
-    };
-    collectd = [{
-      enabled = true;
-      typesdb = "${pkgs.collectd}/share/collectd/types.db";
-      database = "collectd_db";
-      port = 25826;
-    }];
-  };
-
-  krebs.kapacitor =
-    let
-      db = "telegraf_db";
-      echoToIrc = pkgs.writeDash "echo_irc" ''
-        set -euf
-        data="$(${pkgs.jq}/bin/jq -r .message)"
-        export LOGNAME=prism-alarm
-        ${pkgs.irc-announce}/bin/irc-announce \
-          irc.r 6667 prism-alarm \#noise "$data" >/dev/null
-      '';
-    in {
-      enable = true;
-      alarms = {
-        cpu = {
-          database = db;
-          text = ''
-            var data = batch
-              |query(${"'''"}
-                SELECT mean("usage_user") AS mean
-                FROM "${db}"."default"."cpu"
-              ${"'''"})
-              .period(10m)
-              .every(1m)
-              .groupBy('host')
-              data |alert()
-                .crit(lambda: "mean" > 90)
-                .exec('${echoToIrc}')
-              data |deadman(1.0,5m)
-                .stateChangesOnly()
-                .exec('${echoToIrc}')
-          '';
-        };
-        ram = {
-          database = db;
-          text = ''
-            var data = batch
-              |query(${"'''"}
-                SELECT mean("used_percent") AS mean
-                FROM "${db}"."default"."mem"
-              ${"'''"})
-              .period(10m)
-              .every(1m)
-              .groupBy('host')
-              data |alert()
-                .crit(lambda: "mean" > 90)
-                .exec('${echoToIrc}')
-          '';
-        };
-      };
-  };
-
-  services.grafana = {
-    enable = true;
-    addr = "0.0.0.0";
-    auth.anonymous.enable = true;
-    security = import <secrets/grafana_security.nix>; # { AdminUser = ""; adminPassword = ""}
-  };
-
-  krebs.iptables.tables.filter.INPUT.rules = [
-    { predicate = "-p tcp -i retiolum --dport 8086"; target = "ACCEPT"; }
-    { predicate = "-p tcp -i retiolum --dport 3000"; target = "ACCEPT"; }
-    { predicate = "-p udp -i retiolum --dport 25826"; target = "ACCEPT"; }
-  ];
-}