l telegraf: update config

1 files changed, 133 insertions, 42 deletions

diff --git a/lass/2configs/monitoring/telegraf.nix b/lass/2configs/monitoring/telegraf.nix
index 5258b87ed..b172b9c62 100644
--- a/lass/2configs/monitoring/telegraf.nix
+++ b/lass/2configs/monitoring/telegraf.nix
@@ -1,60 +1,127 @@
-{ config, lib, pkgs, ... }:
+{ pkgs, lib, config, ... }:
+# To use this module you also need to allow port 9273 either on the internet or on a vpn interface
+# i.e. networking.firewall.interfaces."vpn0".allowedTCPPorts = [ 9273 ];
+# Example prometheus alert rules:
+# - https://github.com/Mic92/dotfiles/blob/master/nixos/eva/modules/prometheus/alert-rules.nix
 let
   isVM = lib.any (mod: mod == "xen-blkfront" || mod == "virtio_console") config.boot.initrd.kernelModules;
-in {
+  # potentially wrong if the nvme is not used at boot...
+  hasNvme = lib.any (m: m == "nvme") config.boot.initrd.availableKernelModules;
 
-  krebs.iptables.tables.filter.INPUT.rules = [
-    { predicate = "-i retiolum -p tcp --dport 9273"; target = "ACCEPT"; }
-  ];
+  ipv6DadCheck = pkgs.writeShellScript "ipv6-dad-check" ''
+    ${pkgs.iproute2}/bin/ip --json addr | \
+    ${pkgs.jq}/bin/jq -r 'map(.addr_info) | flatten(1) | map(select(.dadfailed == true)) | map(.local) | @text "ipv6_dad_failures count=\(length)i"'
+  '';
 
-  systemd.services.telegraf.path = [ pkgs.nvme-cli ];
+  zfsChecks = lib.optional
+    (lib.any (fs: fs == "zfs") config.boot.supportedFilesystems)
+    (pkgs.writeScript "zpool-health" ''
+      #!${pkgs.gawk}/bin/awk -f
+      BEGIN {
+        while ("${pkgs.zfs}/bin/zpool status" | getline) {
+          if ($1 ~ /pool:/) { printf "zpool_status,name=%s ", $2 }
+          if ($1 ~ /state:/) { printf " state=\"%s\",", $2 }
+          if ($1 ~ /errors:/) {
+              if (index($2, "No")) printf "errors=0i\n"; else printf "errors=%di\n", $2
+          }
+        }
+      }
+    '');
+
+  nfsChecks =
+    let
+      collectHosts = shares: fs:
+        if builtins.elem fs.fsType [ "nfs" "nfs3" "nfs4" ]
+        then
+          shares
+          // (
+            let
+              # also match ipv6 addresses
+              group = builtins.match "\\[?([^\]]+)]?:([^:]+)$" fs.device;
+              host = builtins.head group;
+              path = builtins.elemAt group 1;
+            in
+            {
+              ${host} = (shares.${host} or [ ]) ++ [ path ];
+            }
+          )
+        else shares;
+      nfsHosts = lib.foldl collectHosts { } (builtins.attrValues config.fileSystems);
+    in
+    lib.mapAttrsToList
+      (
+        host: args:
+          (pkgs.writeScript "nfs-health" ''
+            #!${pkgs.gawk}/bin/awk -f
+            BEGIN {
+              for (i = 2; i < ARGC; i++) {
+                  mounts[ARGV[i]] = 1
+              }
+              while ("${pkgs.nfs-utils}/bin/showmount -e " ARGV[1] | getline) {
+                if (NR == 1) { continue }
+                if (mounts[$1] == 1) {
+                    printf "nfs_export,host=%s,path=%s present=1\n", ARGV[1], $1
+                }
+                delete mounts[$1]
+              }
+              for (mount in mounts) {
+                  printf "nfs_export,host=%s,path=%s present=0\n", ARGV[1], $1
+              }
+            }
+          '')
+          + " ${host} ${builtins.concatStringsSep " " args}"
+      )
+      nfsHosts;
+
+in
+{
+
+  systemd.services.telegraf.path = lib.optional (!isVM && hasNvme) pkgs.nvme-cli;
 
   services.telegraf = {
     enable = true;
     extraConfig = {
       agent.interval = "60s";
       inputs = {
-        http_response = [
-          { urls = [
-              "http://localhost:8080/about/health/"
-          ]; }
+        prometheus.urls = lib.mkIf config.services.promtail.enable [
+          # default promtail port
+          "http://localhost:9080/metrics"
         ];
         prometheus.metric_version = 2;
         kernel_vmstat = { };
-        # smart = lib.mkIf (!isVM) {
-        #   path = pkgs.writeShellScript "smartctl" ''
-        #     exec /run/wrappers/bin/sudo ${pkgs.smartmontools}/bin/smartctl "$@"
-        #   '';
-        # };
+        nginx.urls = lib.mkIf config.services.nginx.statusPage [
+          "http://localhost/nginx_status"
+        ];
+        smart = lib.mkIf (!isVM) {
+          path_smartctl = pkgs.writeShellScript "smartctl" ''
+            exec /run/wrappers/bin/sudo ${pkgs.smartmontools}/bin/smartctl "$@"
+          '';
+        };
         system = { };
         mem = { };
-        file = [{
-          data_format = "influx";
-          file_tag = "name";
-          files = [ "/var/log/telegraf/*" ];
-        }] ++ lib.optional (lib.any (fs: fs == "ext4") config.boot.supportedFilesystems) {
-          name_override = "ext4_errors";
-          files = [ "/sys/fs/ext4/*/errors_count" ];
-          data_format = "value";
-        };
-        exec = lib.optionalAttrs (lib.any (fs: fs == "zfs") config.boot.supportedFilesystems) {
-          ## Commands array
-          commands = [
-            (pkgs.writeScript "zpool-health" ''
-              #!${pkgs.gawk}/bin/awk -f
-              BEGIN {
-                while ("${pkgs.zfs}/bin/zpool status" | getline) {
-                  if ($1 ~ /pool:/) { printf "zpool_status,name=%s ", $2 }
-                  if ($1 ~ /state:/) { printf " state=\"%s\",", $2 }
-                  if ($1 ~ /errors:/) {
-                    if (index($2, "No")) printf "errors=0i\n"; else printf "errors=%di\n", $2
-                  }
-                }
-              }
-            '')
-          ];
-          data_format = "influx";
-        };
+        file =
+          [
+            {
+              data_format = "influx";
+              file_tag = "name";
+              files = [ "/var/log/telegraf/*" ];
+            }
+          ]
+          ++ lib.optional (lib.any (fs: fs == "ext4") config.boot.supportedFilesystems) {
+            name_override = "ext4_errors";
+            files = [ "/sys/fs/ext4/*/errors_count" ];
+            data_format = "value";
+          };
+        exec = [
+          {
+            ## Commands array
+            commands =
+              [ ipv6DadCheck ]
+                ++ zfsChecks
+                ++ nfsChecks;
+            data_format = "influx";
+          }
+        ];
         systemd_units = { };
         swap = { };
         disk.tagdrop = {
@@ -62,6 +129,11 @@ in {
           device = [ "rpc_pipefs" "lxcfs" "nsfs" "borgfs" ];
         };
         diskio = { };
+        zfs = {
+          poolMetrics = true;
+        };
+      } // lib.optionalAttrs (if lib.versionAtLeast (lib.versions.majorMinor lib.version) "23.11" then config.boot.swraid.enable else config.boot.initrd.services.swraid.enable) {
+        mdstat = { };
       };
       outputs.prometheus_client = {
         listen = ":9273";
@@ -69,4 +141,23 @@ in {
       };
     };
   };
+  security.sudo.extraRules = lib.mkIf (!isVM) [
+    {
+      users = [ "telegraf" ];
+      commands = [
+        {
+          command = "${pkgs.smartmontools}/bin/smartctl";
+          options = [ "NOPASSWD" ];
+        }
+      ];
+    }
+  ];
+  # avoid logging sudo use
+  security.sudo.configFile = ''
+    Defaults:telegraf !syslog,!pam_session
+  '';
+  # create dummy file to avoid telegraf errors
+  systemd.tmpfiles.rules = [
+    "f /var/log/telegraf/dummy 0444 root root - -"
+  ];
 }