From 1d2c058d78f9cb5d4a51dd2ffb1bfd4b09623055 Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 30 Jan 2017 22:56:43 +0100 Subject: l 2 monitoring: introduce {client,server}.nix --- lass/2configs/monitoring/client.nix | 32 ++++++++++++++++++++ lass/2configs/monitoring/server.nix | 59 +++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 lass/2configs/monitoring/client.nix create mode 100644 lass/2configs/monitoring/server.nix (limited to 'lass/2configs/monitoring') diff --git a/lass/2configs/monitoring/client.nix b/lass/2configs/monitoring/client.nix new file mode 100644 index 00000000..eebf2f2e --- /dev/null +++ b/lass/2configs/monitoring/client.nix @@ -0,0 +1,32 @@ +{pkgs, config, ...}: +with import ; +{ + lass.telegraf = { + enable = true; + outputs = '' + [outputs.influxdb] + urls = ["http://prism:8086"] + database = "all_data" + user_agent = "telegraf" + ''; + inputs = [ + '' + [cpu] + percpu = false + totalcpu = true + drop = ["cpu_time"] + '' + '' + [[inputs.mem]] + '' + '' + [[inputs.ping]] + urls = ["8.8.8.8"] + '' + ]; + }; + systemd.services.telegraf.path = with pkgs; [ + iputils + lm_sensors + ]; +} diff --git a/lass/2configs/monitoring/server.nix b/lass/2configs/monitoring/server.nix new file mode 100644 index 00000000..335820bc --- /dev/null +++ b/lass/2configs/monitoring/server.nix @@ -0,0 +1,59 @@ +{pkgs, config, ...}: +with import ; +{ + services.influxdb = { + enable = true; + }; + + services.influxdb.extraConfig = { + meta.hostname = config.krebs.build.host.name; + # meta.logging-enabled = true; + http.bind-address = ":8086"; + admin.bind-address = ":8083"; + monitoring = { + enabled = false; + # write-interval = "24h"; + }; + }; + + lass.kapacitor = + let + echoToIrc = pkgs.writeDash "echo_irc" '' + set -euf + data="$(${pkgs.jq}/bin/jq -r .message)" + export LOGNAME=prism-alarm + ${pkgs.irc-announce}/bin/irc-announce \ + irc.freenode.org 6667 prism-alarm \#krebs-bots "$data" >/dev/null + ''; + in { + enable = true; + alarms = { + test2 = '' + batch + |query(${"'''"} + SELECT mean("usage_user") AS mean + FROM "${config.lass.kapacitor.check_db}"."default"."cpu" + ${"'''"}) + .every(3m) + .period(1m) + .groupBy('host') + |alert() + .crit(lambda: "mean" > 90) + // Whenever we get an alert write it to a file. + .log('/tmp/alerts.log') + .exec('${echoToIrc}') + ''; + }; + }; + + krebs.iptables.tables.filter.INPUT.rules = [ + { predicate = "-p tcp -i retiolum --dport 8086"; target = "ACCEPT"; } + { predicate = "-p tcp -i retiolum --dport 3000"; target = "ACCEPT"; } + ]; + services.grafana = { + enable = true; + addr = "0.0.0.0"; + auth.anonymous.enable = true; + security = import ; # { AdminUser = ""; adminPassword = ""} + }; +} -- cgit v1.2.3 From 561d4f7bf52a48eca4f4a88c32b4621e5ef3425e Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 3 Feb 2017 00:23:00 +0100 Subject: l 2 monitoring server: add collectdb support --- lass/2configs/monitoring/server.nix | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lass/2configs/monitoring') diff --git a/lass/2configs/monitoring/server.nix b/lass/2configs/monitoring/server.nix index 335820bc..2e1c15ca 100644 --- a/lass/2configs/monitoring/server.nix +++ b/lass/2configs/monitoring/server.nix @@ -14,6 +14,12 @@ with import ; enabled = false; # write-interval = "24h"; }; + collectd = [{ + enabled = true; + typesdb = "${pkgs.collectd}/share/collectd/types.db"; + database = "collectd_db"; + port = 25826; + }]; }; lass.kapacitor = @@ -49,6 +55,7 @@ with import ; krebs.iptables.tables.filter.INPUT.rules = [ { predicate = "-p tcp -i retiolum --dport 8086"; target = "ACCEPT"; } { predicate = "-p tcp -i retiolum --dport 3000"; target = "ACCEPT"; } + { predicate = "-p udp -i retiolum --dport 25826"; target = "ACCEPT"; } ]; services.grafana = { enable = true; -- cgit v1.2.3 From 10b97adc5fb76d7558280f865381f61662cc19c4 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 3 Feb 2017 00:24:21 +0100 Subject: l 2 monitoring client: add more telegraf defaults --- lass/2configs/monitoring/client.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lass/2configs/monitoring') diff --git a/lass/2configs/monitoring/client.nix b/lass/2configs/monitoring/client.nix index eebf2f2e..fc2c2ff4 100644 --- a/lass/2configs/monitoring/client.nix +++ b/lass/2configs/monitoring/client.nix @@ -3,10 +3,13 @@ with import ; { lass.telegraf = { enable = true; + interval = "1s"; + + outputs = '' [outputs.influxdb] urls = ["http://prism:8086"] - database = "all_data" + database = "telegraf_db" user_agent = "telegraf" ''; inputs = [ -- cgit v1.2.3 From 8daef993dcb6149a02c72a4895d6e808a6c6a8d4 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 3 Feb 2017 00:24:39 +0100 Subject: l 2 monitoring client: add collectd config --- lass/2configs/monitoring/client.nix | 59 +++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) (limited to 'lass/2configs/monitoring') diff --git a/lass/2configs/monitoring/client.nix b/lass/2configs/monitoring/client.nix index fc2c2ff4..e879d696 100644 --- a/lass/2configs/monitoring/client.nix +++ b/lass/2configs/monitoring/client.nix @@ -26,10 +26,69 @@ with import ; [[inputs.ping]] urls = ["8.8.8.8"] '' + '' + [[inputs.net]] + '' + '' + [[inputs.dns_query]] + servers = ["8.8.8.8"] + '' ]; }; systemd.services.telegraf.path = with pkgs; [ iputils lm_sensors ]; + + services.collectd = { + enable = true; + autoLoadPlugin = true; + extraConfig = '' + Hostname ${config.krebs.build.host.name} + LoadPlugin load + LoadPlugin disk + LoadPlugin memory + Interval 30.0 + + LoadPlugin interface + + Interface "*Link" + Interface "lo" + Interface "vboxnet*" + Interface "virbr*" + IgnoreSelected true + + + LoadPlugin df + + MountPoint "/nix/store" + FSType "tmpfs" + FSType "binfmt_misc" + FSType "debugfs" + FSType "mqueue" + FSType "hugetlbfs" + FSType "systemd-1" + FSType "cgroup" + FSType "securityfs" + FSType "ramfs" + FSType "proc" + FSType "devpts" + FSType "devtmpfs" + MountPoint "/var/lib/docker/devicemapper" + IgnoreSelected true + + + LoadPlugin cpu + + ReportByCpu true + ReportByState true + ValuesPercentage true + + + LoadPlugin network + + Server "prism" "25826" + + ''; + }; } -- cgit v1.2.3