diff options
| author | makefu <github@syntax-fehler.de> | 2017-02-03 20:45:27 +0100 |
|---|---|---|
| committer | makefu <github@syntax-fehler.de> | 2017-02-04 14:31:42 +0100 |
| commit | d2df5375e705e55764b4cacd4ea32dffcb4c6041 (patch) | |
| tree | 75f993572fdc81ae132efb4d02323ac765e1b9ce | |
| parent | bf405736962fd20df738f84665e5fc7f8d74e72d (diff) | |
m 2 logging: add deadman trigger
| -rw-r--r-- | makefu/2configs/logging/central-stats-server.nix | 36 |
1 files changed, 36 insertions, 0 deletions
diff --git a/makefu/2configs/logging/central-stats-server.nix b/makefu/2configs/logging/central-stats-server.nix index 69e37e493..d4e73ab79 100644 --- a/makefu/2configs/logging/central-stats-server.nix +++ b/makefu/2configs/logging/central-stats-server.nix @@ -6,6 +6,9 @@ let influx-port = 8086; grafana-port = 3000; # TODO nginx forward in { + imports = [ + ../../lass/3modules/kapacitor.nix + ]; services.grafana.enable = true; services.grafana.addr = "0.0.0.0"; @@ -28,6 +31,39 @@ in { port = collectd-port; }]; }; + lass.kapacitor = + let + echoToIrc = pkgs.writeDash "echo_irc" '' + set -euf + data="$(${pkgs.jq}/bin/jq -r .message)" + export LOGNAME=malarm + ${pkgs.irc-announce}/bin/irc-announce \ + irc.freenode.org 6667 malarm \#krebs-bots "$data" >/dev/null + ''; + in { + enable = true; + alarms = { + cpu_deadman = '' + var data = batch + |query(''' + SELECT mean("value") AS mean + FROM "collectd_db"."default"."cpu_value" + WHERE "type_instance" = 'idle' AND "type" = 'percent' fill(0) + ''') + .period(10m) + .every(1m) + .groupBy('host') + data |alert() + .crit(lambda: "mean" < 50) + .stateChangesOnly() + .exec('${echoToIrc}') + data |deadman(1.0,5m) + .stateChangesOnly() + .exec('${echoToIrc}') + ''; + }; + + }; networking.firewall.extraCommands = '' iptables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT iptables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT |