Merge remote-tracking branch 'prism/master'

3 files changed, 182 insertions, 0 deletions

diff --git a/makefu/2configs/stats/client.nix b/makefu/2configs/stats/client.nix
new file mode 100644
index 000000000..dd6ddddaf
--- /dev/null
+++ b/makefu/2configs/stats/client.nix
@@ -0,0 +1,60 @@
+{pkgs, config, ...}:
+{
+  services.collectd = {
+    enable = true;
+    autoLoadPlugin = true;
+    extraConfig = ''
+      Hostname ${config.krebs.build.host.name}
+      LoadPlugin load
+      LoadPlugin disk
+      LoadPlugin memory
+      LoadPlugin df
+      Interval 30.0
+
+      LoadPlugin interface
+      <Plugin "interface">
+        Interface "*Link"
+        Interface "lo"
+        Interface "vboxnet*"
+        Interface "virbr*"
+        IgnoreSelected true
+      </Plugin>
+
+      LoadPlugin df
+      <Plugin "df">
+        MountPoint "/nix/store"
+        # MountPoint "/run*"
+        # MountPoint "/sys*"
+        # MountPoint "/dev"
+        # MountPoint "/dev/shm"
+        # MountPoint "/tmp"
+        FSType "tmpfs"
+        FSType "binfmt_misc"
+        FSType "debugfs"
+        FSType "mqueue"
+        FSType "hugetlbfs"
+        FSType "systemd-1"
+        FSType "cgroup"
+        FSType "securityfs"
+        FSType "ramfs"
+        FSType "proc"
+        FSType "devpts"
+        FSType "devtmpfs"
+        MountPoint "/var/lib/docker/devicemapper"
+        IgnoreSelected true
+      </Plugin>
+
+      LoadPlugin cpu
+      <Plugin cpu>
+        ReportByCpu true
+        ReportByState true
+        ValuesPercentage true
+      </Plugin>
+
+      LoadPlugin network
+      <Plugin "network">
+          Server "${config.makefu.stats-server}" "25826"
+      </Plugin>
+    '';
+  };
+}
diff --git a/makefu/2configs/stats/external/aralast.nix b/makefu/2configs/stats/external/aralast.nix
new file mode 100644
index 000000000..870db99a8
--- /dev/null
+++ b/makefu/2configs/stats/external/aralast.nix
@@ -0,0 +1,38 @@
+{ config, lib, pkgs, ... }:
+
+with import <stockholm/lib>;
+let
+  pkg = pkgs.stdenv.mkDerivation {
+    name = "aralast-master";
+    src = pkgs.fetchFromGitHub {
+      owner = "makefu";
+      repo = "aralast";
+      rev = "7121598";
+      sha256 = "0vw027c698h9b69ksid5p3pji9960hd7n9xi4arrax0vfkwryb4m";
+    };
+    installPhase = ''
+      install -m755 -D aralast.sh $out/bin/aralast
+    '';
+  };
+in {
+  systemd.services.aralast  = {
+    description = "periodically fetch aramark";
+    path = [
+      pkgs.curl
+      pkgs.gnugrep
+      pkgs.gnused
+    ];
+    wantedBy = [ "multi-user.target" ];
+    environment = {
+      INFLUX_HOST = "localhost";
+      INFLUX_PORT = "8086";
+    };
+    # every 10 seconds when the cantina is open
+    startAt = "Mon,Tue,Wed,Thu,Fri *-*-* 6,7,8,9,10,11,12,13,14,15:*:0,15,30,45";
+    serviceConfig = {
+      User = "nobody";
+      ExecStart = "${pkg}/bin/aralast";
+      PrivateTmp = true;
+    };
+  };
+}
diff --git a/makefu/2configs/stats/server.nix b/makefu/2configs/stats/server.nix
new file mode 100644
index 000000000..602fcc6d0
--- /dev/null
+++ b/makefu/2configs/stats/server.nix
@@ -0,0 +1,84 @@
+{pkgs, config, ...}:
+
+with import <stockholm/lib>;
+let
+  collectd-port = 25826;
+  influx-port = 8086;
+  grafana-port = 3000; # TODO nginx forward
+  db = "collectd_db";
+  logging-interface = config.makefu.server.primary-itf;
+in {
+  services.grafana.enable = true;
+  services.grafana.addr = "0.0.0.0";
+
+  services.influxdb.enable = true;
+  # redirect grafana to stats.makefu.r
+  services.nginx.enable = true;
+  services.nginx.virtualHosts."stats.makefu.r".locations."/".proxyPass = "http://localhost:3000";
+  # forward these via nginx
+  services.influxdb.extraConfig = {
+    meta.hostname = config.krebs.build.host.name;
+    # meta.logging-enabled = true;
+    http.bind-address = ":${toString influx-port}";
+    admin.bind-address = ":8083";
+    monitoring = {
+      enabled = false;
+      # write-interval = "24h";
+    };
+    collectd = [{
+      enabled = true;
+      typesdb = "${pkgs.collectd}/share/collectd/types.db";
+      database = db;
+      port = collectd-port;
+    }];
+  };
+  krebs.kapacitor =
+   let
+      echoToIrc = pkgs.writeDash "echo_irc" ''
+        set -euf
+        data="$(${pkgs.jq}/bin/jq -r .message)"
+        export LOGNAME=malarm
+        ${pkgs.irc-announce}/bin/irc-announce \
+          irc.freenode.org 6667 malarm \#krebs-bots "$data" >/dev/null
+      '';
+  in {
+    enable = true;
+    alarms = {
+      cpu_deadman.database = db;
+      cpu_deadman.text = ''
+        var data = batch
+            |query(${"'''"}
+                  SELECT mean("value") AS mean
+                  FROM "collectd_db"."default"."cpu_value"
+                  WHERE "type_instance" = 'idle' AND "type" = 'percent' fill(0)
+                ${"'''"})
+                .period(10m)
+                .every(1m)
+                .groupBy('host')
+        data |alert()
+                .crit(lambda: "mean" < 50)
+                .stateChangesOnly()
+                .exec('${echoToIrc}')
+        data |deadman(1.0,5m)
+                .stateChangesOnly()
+                .exec('${echoToIrc}')
+      '';
+    };
+
+  };
+  networking.firewall.extraCommands = ''
+    iptables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT
+    iptables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT
+    iptables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT
+    iptables -A INPUT -i ${logging-interface} -p udp --dport ${toString collectd-port} -j ACCEPT
+    iptables -A INPUT -i ${logging-interface} -p tcp --dport ${toString influx-port} -j ACCEPT
+    iptables -A INPUT -i ${logging-interface} -p tcp --dport ${toString grafana-port} -j ACCEPT
+
+    ip6tables -A INPUT -i retiolum -p udp --dport ${toString collectd-port} -j ACCEPT
+    ip6tables -A INPUT -i retiolum -p tcp --dport ${toString influx-port} -j ACCEPT
+    ip6tables -A INPUT -i retiolum -p tcp --dport ${toString grafana-port} -j ACCEPT
+    ip6tables -A INPUT -i ${logging-interface} -p udp --dport ${toString collectd-port} -j ACCEPT
+    ip6tables -A INPUT -i ${logging-interface} -p tcp --dport ${toString influx-port} -j ACCEPT
+    ip6tables -A INPUT -i ${logging-interface} -p tcp --dport ${toString grafana-port} -j ACCEPT
+  '';
+}