summaryrefslogtreecommitdiffstats
path: root/lass/2configs/monitoring
diff options
context:
space:
mode:
Diffstat (limited to 'lass/2configs/monitoring')
-rw-r--r--lass/2configs/monitoring/alert-rules.nix208
-rw-r--r--lass/2configs/monitoring/prometheus.nix110
-rw-r--r--lass/2configs/monitoring/telegraf.nix163
3 files changed, 0 insertions, 481 deletions
diff --git a/lass/2configs/monitoring/alert-rules.nix b/lass/2configs/monitoring/alert-rules.nix
deleted file mode 100644
index eae2569f..00000000
--- a/lass/2configs/monitoring/alert-rules.nix
+++ /dev/null
@@ -1,208 +0,0 @@
-# inspiration from https://github.com/Mic92/dotfiles/blob/master/nixos/eva/modules/prometheus/alert-rules.nix
-{ lib }:
-
-lib.mapAttrsToList
- (name: opts: {
- alert = name;
- expr = opts.condition;
- for = opts.time or "2m";
- labels = { };
- annotations.description = opts.description;
- })
- ({
- prometheus_too_many_restarts = {
- condition = ''changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager|telegraf"}[15m]) > 2'';
- description = "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.";
- };
-
- alert_manager_config_not_synced = {
- condition = ''count(count_values("config_hash", alertmanager_config_hash)) > 1'';
- description = "Configurations of AlertManager cluster instances are out of sync.";
- };
-
- prometheus_not_connected_to_alertmanager = {
- condition = "prometheus_notifications_alertmanagers_discovered < 1";
- description = "Prometheus cannot connect the alertmanager\n VALUE = {{ $value }}\n LABELS = {{ $labels }}";
- };
-
- prometheus_rule_evaluation_failures = {
- condition = "increase(prometheus_rule_evaluation_failures_total[3m]) > 0";
- description = "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}";
- };
-
- prometheus_template_expansion_failures = {
- condition = "increase(prometheus_template_text_expansion_failures_total[3m]) > 0";
- time = "0m";
- description = "Prometheus encountered {{ $value }} template text expansion failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}";
- };
-
- promtail_request_errors = {
- condition = ''100 * sum(rate(promtail_request_duration_seconds_count{status_code=~"5..|failed"}[1m])) by (namespace, job, route, instance) / sum(rate(promtail_request_duration_seconds_count[1m])) by (namespace, job, route, instance) > 10'';
- time = "15m";
- description = ''{{ $labels.job }} {{ $labels.route }} is experiencing {{ printf "%.2f" $value }}% errors.'';
- };
-
- promtail_file_lagging = {
- condition = ''abs(promtail_file_bytes_total - promtail_read_bytes_total) > 1e6'';
- time = "15m";
- description = ''{{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} has been lagging by more than 1MB for more than 15m.'';
- };
-
- filesystem_full_80percent = {
- condition = ''disk_used_percent{mode!="ro"} >= 95'';
- time = "10m";
- description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 20% space left on its filesystem.";
- };
-
- filesystem_full_krebs = {
- condition = ''disk_used_percent{mode!="ro", org="krebs"} >= 95'';
- time = "10m";
- description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 5% space left on its filesystem.";
- };
-
- filesystem_inodes_full = {
- condition = ''disk_inodes_free / disk_inodes_total < 0.10'';
- time = "10m";
- description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.path}} got less than 10% inodes left on its filesystem.";
- };
-
- daily_task_not_run = {
- # give 6 hours grace period
- condition = ''time() - task_last_run{state="ok",frequency="daily"} > (24 + 6) * 60 * 60'';
- description = "{{$labels.host}}: {{$labels.name}} was not run in the last 24h";
- };
-
- daily_task_failed = {
- condition = ''task_last_run{state="fail"}'';
- description = "{{$labels.host}}: {{$labels.name}} failed to run";
- };
-
- swap_using_30percent = {
- condition = "mem_swap_total - (mem_swap_cached + mem_swap_free) > mem_swap_total * 0.3";
- time = "30m";
- description = "{{$labels.host}} is using 30% of its swap space for at least 30 minutes.";
- };
-
- systemd_service_failed = {
- condition = ''systemd_units_active_code{name!~"nixpkgs-update-.*.service"} == 3'';
- description = "{{$labels.host}} failed to (re)start service {{$labels.name}}.";
- };
-
- service_not_running = {
- condition = ''systemd_units_active_code{name=~"teamspeak3-server.service|tt-rss.service", sub!="running"}'';
- description = "{{$labels.host}} should have a running {{$labels.name}}.";
- };
-
- nfs_export_not_present = {
- condition = "nfs_export_present == 0";
- time = "1h";
- description = "{{$labels.host}} cannot reach nfs export [{{$labels.server}}]:{{$labels.path}}";
- };
-
- ram_using_90percent = {
- condition = "mem_buffered + mem_free + mem_cached < mem_total * 0.1";
- time = "1h";
- description = "{{$labels.host}} is using at least 90% of its RAM for at least 1 hour.";
- };
- load15 = {
- condition = ''system_load15 / system_n_cpus{org!="nix-community"} >= 2.0'';
- time = "10m";
- description = "{{$labels.host}} is running with load15 > 1 for at least 5 minutes: {{$value}}";
- };
- reboot = {
- condition = "system_uptime < 300";
- description = "{{$labels.host}} just rebooted.";
- };
- uptime = {
- # too scared to upgrade matchbox
- condition = ''system_uptime {host!~"^(matchbox|grandalf)$"} > 2592000'';
- description = "Uptime monster: {{$labels.host}} has been up for more than 30 days.";
- };
- telegraf_down = {
- condition = ''min(up{job=~"telegraf",type!='mobile'}) by (source, job, instance, org) == 0'';
- time = "3m";
- description = "{{$labels.instance}}: {{$labels.job}} telegraf exporter from {{$labels.source}} is down.";
- };
- ping = {
- condition = "ping_result_code{type!='mobile'} != 0";
- description = "{{$labels.url}}: ping from {{$labels.instance}} has failed!";
- };
- ping_high_latency = {
- condition = "ping_average_response_ms{type!='mobile'} > 5000";
- description = "{{$labels.instance}}: ping probe from {{$labels.source}} is encountering high latency!";
- };
- http = {
- condition = "http_response_result_code != 0";
- description = "{{$labels.server}} : http request failed from {{$labels.instance}}: {{$labels.result}}!";
- };
- http_match_failed = {
- condition = "http_response_response_string_match == 0";
- description = "{{$labels.server}} : http body not as expected; status code: {{$labels.status_code}}!";
- };
- dns_query = {
- condition = "dns_query_result_code != 0";
- description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}}!";
- };
- secure_dns_query = {
- condition = "secure_dns_state != 0";
- description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}} for protocol {{$labels.protocol}}!";
- };
- connection_failed = {
- condition = "net_response_result_code != 0";
- description = "{{$labels.server}}: connection to {{$labels.port}}({{$labels.protocol}}) failed from {{$labels.instance}}";
- };
- healthchecks = {
- condition = "hc_check_up == 0";
- description = "{{$labels.instance}}: healtcheck {{$labels.job}} fails!";
- };
- cert_expiry = {
- condition = "x509_cert_expiry < 7*24*3600";
- description = "{{$labels.instance}}: The TLS certificate from {{$labels.source}} will expire in less than 7 days: {{$value}}s";
- };
-
- postfix_queue_length = {
- condition = "avg_over_time(postfix_queue_length[1h]) > 10";
- description = "{{$labels.instance}}: postfix mail queue has undelivered {{$value}} items";
- };
-
- zfs_errors = {
- condition = "zfs_arcstats_l2_io_error + zfs_dmu_tx_error + zfs_arcstats_l2_writes_error > 0";
- description = "{{$labels.instance}} reports: {{$value}} ZFS IO errors.";
- };
-
- # ignore devices that disabled S.M.A.R.T (example if attached via USB)
- smart_errors = {
- condition = ''smart_device_health_ok{enabled!="Disabled"} != 1'';
- description = "{{$labels.instance}}: S.M.A.R.T reports: {{$labels.device}} ({{$labels.model}}) has errors.";
- };
-
- oom_kills = {
- condition = "increase(kernel_vmstat_oom_kill[5m]) > 0";
- description = "{{$labels.instance}}: OOM kill detected";
- };
-
- unusual_disk_read_latency = {
- condition = "rate(diskio_read_time[1m]) / rate(diskio_reads[1m]) > 0.1 and rate(diskio_reads[1m]) > 0";
- description = "{{$labels.instance}}: Disk latency is growing (read operations > 100ms)\n";
- };
-
- unusual_disk_write_latency = {
- condition = "rate(diskio_write_time[1m]) / rate(diskio_write[1m]) > 0.1 and rate(diskio_write[1m]) > 0";
- description = "{{$labels.instance}}: Disk latency is growing (write operations > 100ms)\n";
- };
-
- host_memory_under_memory_pressure = {
- condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
- description = "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
- };
-
- ext4_errors = {
- condition = "ext4_errors_value > 0";
- description = "{{$labels.instance}}: ext4 has reported {{$value}} I/O errors: check /sys/fs/ext4/*/errors_count";
- };
-
- alerts_silences_changed = {
- condition = ''abs(delta(alertmanager_silences{state="active"}[1h])) >= 1'';
- description = "alertmanager: number of active silences has changed: {{$value}}";
- };
- })
diff --git a/lass/2configs/monitoring/prometheus.nix b/lass/2configs/monitoring/prometheus.nix
deleted file mode 100644
index ba32c62a..00000000
--- a/lass/2configs/monitoring/prometheus.nix
+++ /dev/null
@@ -1,110 +0,0 @@
-{ config, lib, pkgs, ... }:
-{
- #prometheus
- krebs.iptables = {
- enable = true;
- tables.filter.INPUT.rules = [
- { predicate = "-i retiolum -p tcp --dport 80"; target = "ACCEPT"; } # nginx
- # { predicate = "-i retiolum -p tcp --dport 3012"; target = "ACCEPT"; } # grafana
- # { predicate = "-i retiolum -p tcp --dport 9093"; target = "ACCEPT"; } # alertmanager
- # { predicate = "-i retiolum -p tcp --dport 9223"; target = "ACCEPT"; } # alertmanager
- ];
- };
-
- services.nginx = {
- enable = true;
- virtualHosts = {
- "prometheus.lass.r" = {
- locations."/".proxyPass = "http://localhost:9090";
- };
- "alert.lass.r" = {
- locations."/".proxyPass = "http://localhost:9093";
- };
- "grafana.lass.r" = {
- locations."/".proxyPass = "http://localhost:3012";
- };
- };
- };
-
- services.grafana = {
- enable = true;
- addr = "0.0.0.0";
- port = 3012;
- auth.anonymous = {
- enable = true;
- org_role = "Admin";
- };
- };
- services.prometheus = {
- enable = true;
- ruleFiles = [
- (pkgs.writeText "prometheus-rules.yml" (builtins.toJSON {
- groups = [{
- name = "alerting-rules";
- rules = import ./alert-rules.nix { inherit lib; };
- }];
- }))
- ];
- scrapeConfigs = [
- {
- job_name = "telegraf";
- scrape_interval = "60s";
- metrics_path = "/metrics";
- static_configs = [
- {
- targets = [
- "prism.r:9273"
- "dishfire.r:9273"
- "yellow.r:9273"
- ];
- }
- ];
- }
- ];
- alertmanagers = [
- { scheme = "http";
- path_prefix = "/";
- static_configs = [ { targets = [ "localhost:9093" ]; } ];
- }
- ];
- alertmanager = {
- enable = true;
- webExternalUrl = "https://alert.lass.r";
- listenAddress = "[::1]";
- configuration = {
- global = {
- # The smarthost and SMTP sender used for mail notifications.
- smtp_smarthost = "localhost:587";
- smtp_from = "alertmanager@alert.lass.r";
- # smtp_auth_username = "alertmanager@thalheim.io";
- # smtp_auth_password = "$SMTP_PASSWORD";
- };
- route = {
- receiver = "default";
- routes = [
- {
- group_by = [ "host" ];
- group_wait = "30s";
- group_interval = "2m";
- repeat_interval = "2h";
- receiver = "all";
- }
- ];
- };
- receivers = [
- {
- name = "all";
- webhook_configs = [{
- url = "http://127.0.0.1:9223/";
- max_alerts = 5;
- }];
- }
- {
- name = "default";
- }
- ];
- };
- };
- };
-
-}
diff --git a/lass/2configs/monitoring/telegraf.nix b/lass/2configs/monitoring/telegraf.nix
deleted file mode 100644
index b172b9c6..00000000
--- a/lass/2configs/monitoring/telegraf.nix
+++ /dev/null
@@ -1,163 +0,0 @@
-{ pkgs, lib, config, ... }:
-# To use this module you also need to allow port 9273 either on the internet or on a vpn interface
-# i.e. networking.firewall.interfaces."vpn0".allowedTCPPorts = [ 9273 ];
-# Example prometheus alert rules:
-# - https://github.com/Mic92/dotfiles/blob/master/nixos/eva/modules/prometheus/alert-rules.nix
-let
- isVM = lib.any (mod: mod == "xen-blkfront" || mod == "virtio_console") config.boot.initrd.kernelModules;
- # potentially wrong if the nvme is not used at boot...
- hasNvme = lib.any (m: m == "nvme") config.boot.initrd.availableKernelModules;
-
- ipv6DadCheck = pkgs.writeShellScript "ipv6-dad-check" ''
- ${pkgs.iproute2}/bin/ip --json addr | \
- ${pkgs.jq}/bin/jq -r 'map(.addr_info) | flatten(1) | map(select(.dadfailed == true)) | map(.local) | @text "ipv6_dad_failures count=\(length)i"'
- '';
-
- zfsChecks = lib.optional
- (lib.any (fs: fs == "zfs") config.boot.supportedFilesystems)
- (pkgs.writeScript "zpool-health" ''
- #!${pkgs.gawk}/bin/awk -f
- BEGIN {
- while ("${pkgs.zfs}/bin/zpool status" | getline) {
- if ($1 ~ /pool:/) { printf "zpool_status,name=%s ", $2 }
- if ($1 ~ /state:/) { printf " state=\"%s\",", $2 }
- if ($1 ~ /errors:/) {
- if (index($2, "No")) printf "errors=0i\n"; else printf "errors=%di\n", $2
- }
- }
- }
- '');
-
- nfsChecks =
- let
- collectHosts = shares: fs:
- if builtins.elem fs.fsType [ "nfs" "nfs3" "nfs4" ]
- then
- shares
- // (
- let
- # also match ipv6 addresses
- group = builtins.match "\\[?([^\]]+)]?:([^:]+)$" fs.device;
- host = builtins.head group;
- path = builtins.elemAt group 1;
- in
- {
- ${host} = (shares.${host} or [ ]) ++ [ path ];
- }
- )
- else shares;
- nfsHosts = lib.foldl collectHosts { } (builtins.attrValues config.fileSystems);
- in
- lib.mapAttrsToList
- (
- host: args:
- (pkgs.writeScript "nfs-health" ''
- #!${pkgs.gawk}/bin/awk -f
- BEGIN {
- for (i = 2; i < ARGC; i++) {
- mounts[ARGV[i]] = 1
- }
- while ("${pkgs.nfs-utils}/bin/showmount -e " ARGV[1] | getline) {
- if (NR == 1) { continue }
- if (mounts[$1] == 1) {
- printf "nfs_export,host=%s,path=%s present=1\n", ARGV[1], $1
- }
- delete mounts[$1]
- }
- for (mount in mounts) {
- printf "nfs_export,host=%s,path=%s present=0\n", ARGV[1], $1
- }
- }
- '')
- + " ${host} ${builtins.concatStringsSep " " args}"
- )
- nfsHosts;
-
-in
-{
-
- systemd.services.telegraf.path = lib.optional (!isVM && hasNvme) pkgs.nvme-cli;
-
- services.telegraf = {
- enable = true;
- extraConfig = {
- agent.interval = "60s";
- inputs = {
- prometheus.urls = lib.mkIf config.services.promtail.enable [
- # default promtail port
- "http://localhost:9080/metrics"
- ];
- prometheus.metric_version = 2;
- kernel_vmstat = { };
- nginx.urls = lib.mkIf config.services.nginx.statusPage [
- "http://localhost/nginx_status"
- ];
- smart = lib.mkIf (!isVM) {
- path_smartctl = pkgs.writeShellScript "smartctl" ''
- exec /run/wrappers/bin/sudo ${pkgs.smartmontools}/bin/smartctl "$@"
- '';
- };
- system = { };
- mem = { };
- file =
- [
- {
- data_format = "influx";
- file_tag = "name";
- files = [ "/var/log/telegraf/*" ];
- }
- ]
- ++ lib.optional (lib.any (fs: fs == "ext4") config.boot.supportedFilesystems) {
- name_override = "ext4_errors";
- files = [ "/sys/fs/ext4/*/errors_count" ];
- data_format = "value";
- };
- exec = [
- {
- ## Commands array
- commands =
- [ ipv6DadCheck ]
- ++ zfsChecks
- ++ nfsChecks;
- data_format = "influx";
- }
- ];
- systemd_units = { };
- swap = { };
- disk.tagdrop = {
- fstype = [ "tmpfs" "ramfs" "devtmpfs" "devfs" "iso9660" "overlay" "aufs" "squashfs" ];
- device = [ "rpc_pipefs" "lxcfs" "nsfs" "borgfs" ];
- };
- diskio = { };
- zfs = {
- poolMetrics = true;
- };
- } // lib.optionalAttrs (if lib.versionAtLeast (lib.versions.majorMinor lib.version) "23.11" then config.boot.swraid.enable else config.boot.initrd.services.swraid.enable) {
- mdstat = { };
- };
- outputs.prometheus_client = {
- listen = ":9273";
- metric_version = 2;
- };
- };
- };
- security.sudo.extraRules = lib.mkIf (!isVM) [
- {
- users = [ "telegraf" ];
- commands = [
- {
- command = "${pkgs.smartmontools}/bin/smartctl";
- options = [ "NOPASSWD" ];
- }
- ];
- }
- ];
- # avoid logging sudo use
- security.sudo.configFile = ''
- Defaults:telegraf !syslog,!pam_session
- '';
- # create dummy file to avoid telegraf errors
- systemd.tmpfiles.rules = [
- "f /var/log/telegraf/dummy 0444 root root - -"
- ];
-}
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-19 23:05:44 +0000