puyak: enable blackbox prometheus exporter, use it to ping internal and external hosts

2 files changed, 60 insertions, 13 deletions

diff --git a/krebs/2configs/shack/prometheus/blackbox.nix b/krebs/2configs/shack/prometheus/blackbox.nix
new file mode 100644
index 000000000..82ce003e8
--- /dev/null
+++ b/krebs/2configs/shack/prometheus/blackbox.nix
@@ -0,0 +1,19 @@
+{pkgs, ... }:
+{
+  systemd.services.prometheus-blackbox-exporter.serviceConfig = {
+    CapabilityBoundingSet = ["CAP_NET_RAW"]; # icmp allow
+    AmbientCapabilities = ["CAP_NET_RAW"];
+  };
+  services.prometheus.exporters.blackbox = {
+    enable = true;
+    # openFirewall = true; # not requred if running on the same host as prometheus
+    port = 9115;
+    configFile = pkgs.writeText "icmp" ''
+      modules:
+        icmp:
+          prober: icmp
+          icmp:
+            preferred_ip_protocol: ip4
+    '';
+  };
+}
diff --git a/krebs/2configs/shack/prometheus/server.nix b/krebs/2configs/shack/prometheus/server.nix
index f5d2e7640..c088a3b08 100644
--- a/krebs/2configs/shack/prometheus/server.nix
+++ b/krebs/2configs/shack/prometheus/server.nix
@@ -18,16 +18,6 @@
     };
     prometheus = {
       enable = true;
-      extraFlags = [
-        "-storage.local.retention 720h"
-        "-storage.local.series-file-shrink-ratio 0.3"
-        "-storage.local.memory-chunks 2097152"
-        "-storage.local.max-chunks-to-persist 1048576"
-        "-storage.local.index-cache-size.fingerprint-to-metric 2097152"
-        "-storage.local.index-cache-size.fingerprint-to-timerange 1048576"
-        "-storage.local.index-cache-size.label-name-to-label-values 2097152"
-        "-storage.local.index-cache-size.label-pair-to-fingerprints 41943040"
-      ];
       ruleFiles = lib.singleton (pkgs.writeText "prometheus-rules.yml" (builtins.toJSON {
             groups = lib.singleton {
               name = "mf-alerting-rules";
@@ -41,7 +31,7 @@
           static_configs = [
             {
               targets = [
-                "localhost:9100"
+                "wolf.shack:9100"
               ];
               labels = {
                 alias = "wolf.shack";
@@ -49,7 +39,15 @@
             }
             {
               targets = [
-                "localhost:9130"
+                "infra01.shack:9100"
+              ];
+              labels = {
+                alias = "infra01.shack";
+              };
+            }
+            {
+              targets = [
+                "unifi.shack:9130"
               ];
               labels = {
                 alias = "unifi.shack";
@@ -57,7 +55,7 @@
             }
             {
               targets = [
-                "10.42.22.184:9100" # puyak.shack
+                "puyak.shack:9100" # puyak.shack
               ];
               labels = {
                 alias = "puyak.shack";
@@ -81,6 +79,36 @@
             }
           ];
         }
+        {
+          job_name = "blackbox";
+          metrics_path = "/probe";
+          params.module = [ "icmp" ];
+          static_configs = [
+            {
+              targets = [
+                "google.com"
+                "wolf.shack"
+                "web.de"
+                "10.0.0.1"
+                "licht.shack"
+              ];
+            }
+          ];
+          relabel_configs = [
+            {
+              source_labels = ["__address__"];
+              target_label = "__param_target";
+            }
+            {
+              source_labels = ["__param_target"];
+              target_label = "instance";
+            }
+            {
+              target_label = "__address__";
+              replacement = "127.0.0.1:9115";
+            }
+          ];
+        }
       ];
       alertmanagers = [
         { scheme = "http";