7 files changed, 73 insertions, 6 deletions
diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index cd84ab815..1e63a26e2 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -385,6 +385,8 @@ with import <stockholm/lib>;
           aliases = [
             "omo.retiolum"
             "omo.r"
+            "logs.makefu.r"
+            "stats.makefu.r"
           ];
           tinc.pubkey = ''
               -----BEGIN RSA PUBLIC KEY-----
diff --git a/lass/3modules/kapacitor.nix b/lass/3modules/kapacitor.nix
index a1e82b830..299a56d92 100644
--- a/lass/3modules/kapacitor.nix
+++ b/lass/3modules/kapacitor.nix
@@ -21,6 +21,10 @@ let
       type = types.str;
       default = "kapacitor";
     };
+    logLevel = mkOption {
+      type = types.enum ["DEBUG" "INFO" "WARN" "ERROR" "OFF"];
+      default = "INFO";
+    };
     alarms = mkOption {
       type = with types; attrsOf str;
       default = {};
@@ -82,7 +86,7 @@ let
 
         [logging]
           file = "STDERR"
-          level = "DEBUG"
+          level = "${cfg.logLevel}"
 
         [deadman]
           interval = "10s"
diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix
index 7a36c57be..a6a336fed 100644
--- a/makefu/1systems/omo.nix
+++ b/makefu/1systems/omo.nix
@@ -54,9 +54,10 @@ in {
       ../2configs/omo-share.nix
       ../2configs/tinc/retiolum.nix
       ../2configs/logging/central-stats-server.nix
+      ../2configs/logging/central-logging-server.nix
       ../2configs/logging/central-stats-client.nix
+      ../2configs/logging/central-logging-client.nix
 
-      ../2configs/kapacitor.nix
       # ../2configs/torrent.nix
 
       # ../2configs/elchos/search.nix
diff --git a/makefu/2configs/logging/central-logging-client.nix b/makefu/2configs/logging/central-logging-client.nix
new file mode 100644
index 000000000..04d2de0d0
--- /dev/null
+++ b/makefu/2configs/logging/central-logging-client.nix
@@ -0,0 +1,32 @@
+{pkgs, buil, config, ...}:
+let
+  log-server = config.makefu.log-server;
+  log-port = 9200;
+in {
+  services.journalbeat = {
+    enable = true;
+    # TODO: filter for certain journal fields, not all
+    extraConfig = ''
+      journalbeat:
+        name: logs-${config.krebs.build.host.name}
+        seek_position: cursor
+        cursor_seek_fallback: tail
+        write_cursor_state: true
+        cursor_flush_period: 5s
+        clean_field_names: true
+        convert_to_numbers: false
+        move_metadata_to_field: journal
+        default_type: journal
+      output.elasticsearch:
+        enabled: true
+        hosts: ["${log-server}:${builtins.toString log-port}"]
+        template.enabled: false
+      #output.console:
+      #  enabled: true
+      logging.level: info
+      logging.to_syslog: true
+      logging.selectors: ["*"]
+
+    '';
+  };
+}
diff --git a/makefu/2configs/logging/central-logging-server.nix b/makefu/2configs/logging/central-logging-server.nix
new file mode 100644
index 000000000..e2cfe6948
--- /dev/null
+++ b/makefu/2configs/logging/central-logging-server.nix
@@ -0,0 +1,23 @@
+{pkgs, config, ...}:
+
+with import <stockholm/lib>;
+let
+  es-port = 9200;
+  kibana-port = 5601;
+in {
+  services.elasticsearch = {
+    enable = true;
+    listenAddress = "0.0.0.0";
+    port = es-port;
+  };
+  services.kibana = {
+    enable = true;
+    listenAddress = "0.0.0.0";
+    port = kibana-port;
+  };
+
+  networking.firewall.extraCommands = ''
+    iptables -A INPUT -i retiolum -p tcp --dport ${toString es-port} -j ACCEPT
+    iptables -A INPUT -i retiolum -p tcp --dport ${toString es-port} -j ACCEPT
+  '';
+}
diff --git a/makefu/2configs/logging/central-stats-server.nix b/makefu/2configs/logging/central-stats-server.nix
index d4e73ab79..8151d4939 100644
--- a/makefu/2configs/logging/central-stats-server.nix
+++ b/makefu/2configs/logging/central-stats-server.nix
@@ -7,7 +7,7 @@ let
   grafana-port = 3000; # TODO nginx forward
 in {
   imports = [
-    ../../lass/3modules/kapacitor.nix
+    ../../../lass/3modules/kapacitor.nix
   ];
   services.grafana.enable = true;
   services.grafana.addr = "0.0.0.0";
@@ -45,11 +45,11 @@ in {
     alarms = {
       cpu_deadman = ''
         var data = batch
-            |query('''
+            |query(${"'''"}
                   SELECT mean("value") AS mean
                   FROM "collectd_db"."default"."cpu_value"
                   WHERE "type_instance" = 'idle' AND "type" = 'percent' fill(0)
-                ''')
+                ${"'''"})
                 .period(10m)
                 .every(1m)
                 .groupBy('host')
diff --git a/makefu/3modules/logging-config.nix b/makefu/3modules/logging-config.nix
index cd6924f61..a77e32e48 100644
--- a/makefu/3modules/logging-config.nix
+++ b/makefu/3modules/logging-config.nix
@@ -4,8 +4,13 @@ with import <stockholm/lib>;
 {
   options.makefu.stats-server = lib.mkOption {
     type = types.str;
-    default = "omo.retiolum";
+    default = "stats.makefu.r";
     description = "Central stats server (collectd)";
   };
+  options.makefu.log-server = lib.mkOption {
+    type = types.str;
+    default = "logs.makefu.r";
+    description = "Central logging server (logstash,elasticsearch)";
+  };
 }