From 0c92dd719a46139523f6e353c354871bd78024a4 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 25 May 2017 23:19:36 +0200 Subject: m 2: rename stats and share --- makefu/2configs/stats/client.nix | 60 +++++++++++++++++++++ makefu/2configs/stats/external/aralast.nix | 38 ++++++++++++++ makefu/2configs/stats/server.nix | 84 ++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+) create mode 100644 makefu/2configs/stats/client.nix create mode 100644 makefu/2configs/stats/external/aralast.nix create mode 100644 makefu/2configs/stats/server.nix (limited to 'makefu/2configs/stats') diff --git a/makefu/2configs/stats/client.nix b/makefu/2configs/stats/client.nix new file mode 100644 index 000000000..dd6ddddaf --- /dev/null +++ b/makefu/2configs/stats/client.nix @@ -0,0 +1,60 @@ +{pkgs, config, ...}: +{ + services.collectd = { + enable = true; + autoLoadPlugin = true; + extraConfig = '' + Hostname ${config.krebs.build.host.name} + LoadPlugin load + LoadPlugin disk + LoadPlugin memory + LoadPlugin df + Interval 30.0 + + LoadPlugin interface + + Interface "*Link" + Interface "lo" + Interface "vboxnet*" + Interface "virbr*" + IgnoreSelected true + + + LoadPlugin df + + MountPoint "/nix/store" + # MountPoint "/run*" + # MountPoint "/sys*" + # MountPoint "/dev" + # MountPoint "/dev/shm" + # MountPoint "/tmp" + FSType "tmpfs" + FSType "binfmt_misc" + FSType "debugfs" + FSType "mqueue" + FSType "hugetlbfs" + FSType "systemd-1" + FSType "cgroup" + FSType "securityfs" + FSType "ramfs" + FSType "proc" + FSType "devpts" + FSType "devtmpfs" + MountPoint "/var/lib/docker/devicemapper" + IgnoreSelected true + + + LoadPlugin cpu + + ReportByCpu true + ReportByState true + ValuesPercentage true + + + LoadPlugin network + + Server "${config.makefu.stats-server}" "25826" + + ''; + }; +} diff --git a/makefu/2configs/stats/external/aralast.nix b/makefu/2configs/stats/external/aralast.nix new file mode 100644 index 000000000..c335db454 --- /dev/null +++ b/makefu/2configs/stats/external/aralast.nix @@ -0,0 +1,38 @@ +{ config, lib, pkgs, ... }: + +with import ; +let + pkg = pkgs.stdenv.mkDerivation { + name = "aralast-master"; + src = pkgs.fetchFromGitHub { + owner = "makefu"; + repo = "aralast"; + rev = "7121598"; + sha256 = "0vw027c698h9b69ksid5p3pji9960hd7n9xi4arrax0vfkwryb4m"; + }; + installPhase = '' + install -m755 -D aralast.sh $out/bin/aralast + ''; + }; +in { + systemd.services.aralast = { + description = "periodically fetch aramark"; + path = [ + pkgs.curl + pkgs.gnugrep + pkgs.gnused + ]; + wantedBy = [ "multi-user.target" ]; + environment = { + INFLUX_HOST = "localhost"; + INFLUX_PORT = "8086"; + }; + # every 10 seconds when the cantina is open + startAt = "Mon,Tue,Wed,Thu,Fri *-*-* 6,7,8,9,10,11,12,13,14,15:*:0/10"; + serviceConfig = { + User = "nobody"; + ExecStart = "${pkg}/bin/aralast"; + PrivateTmp = true; + }; + }; +} diff --git a/makefu/2configs/stats/server.nix b/makefu/2configs/stats/server.nix new file mode 100644 index 000000000..602fcc6d0 --- /dev/null +++ b/makefu/2configs/stats/server.nix @@ -0,0 +1,84 @@ +{pkgs, config, ...}: + +with import ; +let + collectd-port = 25826; + influx-port = 8086; + grafana-port = 3000; # TODO nginx forward + db = "collectd_db"; + logging-interface = config.makefu.server.primary-itf; +in { + services.grafana.enable = true; + services.grafana.addr = "0.0.0.0"; + + services.influxdb.enable = true; + # redirect grafana to stats.makefu.r + services.nginx.enable = true; + services.nginx.virtualHosts."stats.makefu.r".locations."/".proxyPass = "http://localhost:3000"; + # forward these via nginx + services.influxdb.extraConfig = { + meta.hostname = config.krebs.build.host.name; + # meta.logging-enabled = true; + http.bind-address = ":${toString influx-port}"; + admin.bind-address = ":8083"; + monitoring = { + enabled = false; + # write-interval = "24h"; + }; + collectd = [{ + enabled = true; + typesdb = "${pkgs.collectd}/share/collectd/types.db"; + database = db; + port = collectd-port; + }]; + }; + krebs.kapacitor = + let + echoToIrc = pkgs.writeDash "echo_irc" '' + set -euf + data="$(${pkgs.jq}/bin/jq -r .message)" + export LOGNAME=malarm + ${pkgs.irc-announce}/bin/irc-announce \ + irc.freenode.org 6667 malarm \#krebs-bots "$data" >/dev/null + ''; + in { + enable = true; + alarms = { + cpu_deadman.database = db; + cpu_deadman.text = '' + var data = batch + |query(${"'''"} + SELECT mean("value") AS mean + FROM "collectd_db"."default"."cpu_value" + WHERE "type_instance" = 'idle' AND "type" = 'percent' fill(0) + ${"'''"}) + .period(10m) + .every(1m) + .groupBy('host') + data |alert() + .crit(lambda: "mean" < 50) + .stateChangesOnly() + .exec('${echoToIrc}') + data |deadman(1.0,5m) + .stateChangesOnly() + .exec('${echoToIrc}') + ''; + }; + + }; + networking.firewall.extraCommands = '' + iptables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT + iptables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT + iptables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT + iptables -A INPUT -i ${logging-interface} -p udp --dport ${toString collectd-port} -j ACCEPT + iptables -A INPUT -i ${logging-interface} -p tcp --dport ${toString influx-port} -j ACCEPT + iptables -A INPUT -i ${logging-interface} -p tcp --dport ${toString grafana-port} -j ACCEPT + + ip6tables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT + ip6tables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT + ip6tables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT + ip6tables -A INPUT -i ${logging-interface} -p udp --dport ${toString collectd-port} -j ACCEPT + ip6tables -A INPUT -i ${logging-interface} -p tcp --dport ${toString influx-port} -j ACCEPT + ip6tables -A INPUT -i ${logging-interface} -p tcp --dport ${toString grafana-port} -j ACCEPT + ''; +} -- cgit v1.2.3 From 28b51692a2f2fe22301954d1ae1b4fd36fc3f8bb Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 1 Jun 2017 10:14:01 +0200 Subject: m 2 aralast:fix timing --- makefu/2configs/stats/external/aralast.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'makefu/2configs/stats') diff --git a/makefu/2configs/stats/external/aralast.nix b/makefu/2configs/stats/external/aralast.nix index c335db454..870db99a8 100644 --- a/makefu/2configs/stats/external/aralast.nix +++ b/makefu/2configs/stats/external/aralast.nix @@ -28,7 +28,7 @@ in { INFLUX_PORT = "8086"; }; # every 10 seconds when the cantina is open - startAt = "Mon,Tue,Wed,Thu,Fri *-*-* 6,7,8,9,10,11,12,13,14,15:*:0/10"; + startAt = "Mon,Tue,Wed,Thu,Fri *-*-* 6,7,8,9,10,11,12,13,14,15:*:0,15,30,45"; serviceConfig = { User = "nobody"; ExecStart = "${pkg}/bin/aralast"; -- cgit v1.2.3