diff options
Diffstat (limited to 'krebs/2configs/shack/prometheus/server.nix')
| -rw-r--r-- | krebs/2configs/shack/prometheus/server.nix | 162 |
1 files changed, 56 insertions, 106 deletions
diff --git a/krebs/2configs/shack/prometheus/server.nix b/krebs/2configs/shack/prometheus/server.nix index 12f757e8..9e4b4d1a 100644 --- a/krebs/2configs/shack/prometheus/server.nix +++ b/krebs/2configs/shack/prometheus/server.nix @@ -1,6 +1,9 @@ { pkgs, lib, config, ... }: # from https://gist.github.com/globin/02496fd10a96a36f092a8e7ea0e6c7dd { + imports = [ + ./alert-rules.nix + ]; networking = { firewall.allowedTCPPorts = [ 9090 # prometheus @@ -18,101 +21,6 @@ }; prometheus = { enable = true; - extraFlags = [ - "-storage.local.retention 720h" - "-storage.local.series-file-shrink-ratio 0.3" - "-storage.local.memory-chunks 2097152" - "-storage.local.max-chunks-to-persist 1048576" - "-storage.local.index-cache-size.fingerprint-to-metric 2097152" - "-storage.local.index-cache-size.fingerprint-to-timerange 1048576" - "-storage.local.index-cache-size.label-name-to-label-values 2097152" - "-storage.local.index-cache-size.label-pair-to-fingerprints 41943040" - ]; - alertmanagerURL = [ "http://localhost:9093" ]; - rules = [ - '' - ALERT node_down - IF up == 0 - FOR 5m - LABELS { - severity="page" - } - ANNOTATIONS { - summary = "{{$labels.alias}}: Node is down.", - description = "{{$labels.alias}} has been down for more than 5 minutes." - } - ALERT node_systemd_service_failed - IF node_systemd_unit_state{state="failed"} == 1 - FOR 4m - LABELS { - severity="page" - } - ANNOTATIONS { - summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start.", - description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}." - } - ALERT node_filesystem_full_90percent - IF sort(node_filesystem_free{device!="ramfs"} < node_filesystem_size{device!="ramfs"} * 0.1) / 1024^3 - FOR 5m - LABELS { - severity="page" - } - ANNOTATIONS { - summary = "{{$labels.alias}}: Filesystem is running out of space soon.", - description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% space left on its filesystem." - } - ALERT node_filesystem_full_in_4h - IF predict_linear(node_filesystem_free{device!="ramfs"}[1h], 4*3600) <= 0 - FOR 5m - LABELS { - severity="page" - } - ANNOTATIONS { - summary = "{{$labels.alias}}: Filesystem is running out of space in 4 hours.", - description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 4 hours" - } - ALERT node_filedescriptors_full_in_3h - IF predict_linear(node_filefd_allocated[1h], 3*3600) >= node_filefd_maximum - FOR 20m - LABELS { - severity="page" - } - ANNOTATIONS { - summary = "{{$labels.alias}} is running out of available file descriptors in 3 hours.", - description = "{{$labels.alias}} is running out of available file descriptors in approx. 3 hours" - } - ALERT node_load1_90percent - IF node_load1 / on(alias) count(node_cpu{mode="system"}) by (alias) >= 0.9 - FOR 1h - LABELS { - severity="page" - } - ANNOTATIONS { - summary = "{{$labels.alias}}: Running on high load.", - description = "{{$labels.alias}} is running with > 90% total load for at least 1h." - } - ALERT node_cpu_util_90percent - IF 100 - (avg by (alias) (irate(node_cpu{mode="idle"}[5m])) * 100) >= 90 - FOR 1h - LABELS { - severity="page" - } - ANNOTATIONS { - summary = "{{$labels.alias}}: High CPU utilization.", - description = "{{$labels.alias}} has total CPU utilization over 90% for at least 1h." - } - ALERT node_ram_using_90percent - IF node_memory_MemFree + node_memory_Buffers + node_memory_Cached < node_memory_MemTotal * 0.1 - FOR 30m - LABELS { - severity="page" - } - ANNOTATIONS { - summary="{{$labels.alias}}: Using lots of RAM.", - description="{{$labels.alias}} is using at least 90% of its RAM for at least 30 minutes now.", - } - '' - ]; scrapeConfigs = [ { job_name = "node"; @@ -120,7 +28,7 @@ static_configs = [ { targets = [ - "localhost:9100" + "wolf.shack:9100" ]; labels = { alias = "wolf.shack"; @@ -128,7 +36,15 @@ } { targets = [ - "localhost:9130" + "infra01.shack:9100" + ]; + labels = { + alias = "infra01.shack"; + }; + } + { + targets = [ + "unifi.shack:9130" ]; labels = { alias = "unifi.shack"; @@ -136,7 +52,7 @@ } { targets = [ - "10.42.22.184:9100" # puyak.shack + "puyak.shack:9100" # puyak.shack ]; labels = { alias = "puyak.shack"; @@ -160,10 +76,49 @@ } ]; } + { + job_name = "blackbox"; + metrics_path = "/probe"; + params.module = [ "icmp" ]; + static_configs = [ + { + targets = [ + "google.com" + "wolf.shack" + "web.de" + "10.0.0.1" + "licht.shack" + ]; + } + ]; + relabel_configs = [ + { + source_labels = ["__address__"]; + target_label = "__param_target"; + } + { + source_labels = ["__param_target"]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = "127.0.0.1:9115"; + } + ]; + } + ]; + alertmanagers = [ + { scheme = "http"; + path_prefix = "/"; + static_configs = [ { targets = [ "localhost:9093" ]; } ]; + } ]; alertmanager = { enable = true; - listenAddress = "0.0.0.0"; + listenAddress = "127.0.0.1"; + webExternalUrl = "http://alert.prometheus.shack"; + logLevel = "debug"; + configuration = { "global" = { "smtp_smarthost" = "smtp.example.com:587"; @@ -179,15 +134,10 @@ "receivers" = [ { "name" = "team-admins"; - "email_configs" = [ - { - "to" = "devnull@example.com"; - "send_resolved" = true; - } - ]; + "email_configs" = [ ]; "webhook_configs" = [ { - "url" = "https://example.com/prometheus-alerts"; + "url" = "http://localhost:16320"; "send_resolved" = true; } ]; |