diff options
| -rw-r--r-- | makefu/2configs/logging/filter/dnsmasq.conf | 19 | ||||
| -rw-r--r-- | makefu/2configs/logging/patterns/dnsmasq | 15 | ||||
| -rw-r--r-- | makefu/2configs/logging/server.nix | 123 |
3 files changed, 154 insertions, 3 deletions
diff --git a/makefu/2configs/logging/filter/dnsmasq.conf b/makefu/2configs/logging/filter/dnsmasq.conf new file mode 100644 index 000000000..1570b1c60 --- /dev/null +++ b/makefu/2configs/logging/filter/dnsmasq.conf @@ -0,0 +1,19 @@ + +if ( [program] == "dnsmasq") { + grok { + patterns_dir => ["${./patterns}"] + match => { + "message" => [ + "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: query\[[\w]+\] %{domain:DOMAIN} from %{clientip:CLIENTIP}" + , "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: reply %{domain:DOMAIN} is %{ip:IP}" + , "^%{logdate:LOGDATE} dnsmasq\[[\d]+\]\: %{blocklist:BLOCKLIST} %{domain:DOMAIN} is %{ip:IP}" + ] + } + } + date { + match => [ "LOGDATE", "MMM dd HH:mm:ss", "MMM d HH:mm:ss", "ISO8601" ] + } + geoip { + source => "IP" + } +} diff --git a/makefu/2configs/logging/patterns/dnsmasq b/makefu/2configs/logging/patterns/dnsmasq new file mode 100644 index 000000000..c1e700d5e --- /dev/null +++ b/makefu/2configs/logging/patterns/dnsmasq @@ -0,0 +1,15 @@ +BLOCKLIST [\/\w\.]+ +DOMAIN [\w\.\-]+ +DNSID \d+ +PORT \d+ +DNSRESPONSE cached|reply|forwarded|query +# TODO: there are some strange responses for certain queries like <CNAME> or ... +IPORWORD %{IP}|[<>\.\/\w>]+ + +# TODO use public suffix list by mozilla +TLD [a-z]{2,63} +# matches CCSLD and TLD together (e.g. co.uk ) +CCSLD_TLD [a-z]+\.uk +# actually after a CCTLD this would be the third level domain ... +PUBLIC_SUFFIX (xn--)?%{FUNCTIONAL_SLD}\.(%{CCSLD_TLD}|%{TLD}) +FUNCTIONAL_SLD [a-z0-9-]{1,63} diff --git a/makefu/2configs/logging/server.nix b/makefu/2configs/logging/server.nix index 90f8e6680..f2fccec25 100644 --- a/makefu/2configs/logging/server.nix +++ b/makefu/2configs/logging/server.nix @@ -1,18 +1,29 @@ {pkgs, config, ...}: -with import <stockholm/lib>; let es-port = 9200; kibana-port = 5601; + primaryName = "log.${config.krebs.build.host.name}"; + serverAliases = [ "${primaryName}.r" "${primaryName}.lan" ]; in { + + services.nginx.virtualHosts.${primaryName} = { + inherit serverAliases; + locations."/" = { + proxyPass = "http://localhost:5601/"; + extraConfig = '' + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + ''; + }; + }; services.elasticsearch = { enable = true; - listenAddress = "0.0.0.0"; port = es-port; }; services.kibana = { enable = true; - listenAddress = "0.0.0.0"; port = kibana-port; }; @@ -20,4 +31,110 @@ in { iptables -A INPUT -i retiolum -p tcp --dport ${toString es-port} -j ACCEPT iptables -A INPUT -i retiolum -p tcp --dport ${toString kibana-port} -j ACCEPT ''; + + # send logs directly to elasticsearch + services.journalbeat = { + enable = true; + package = pkgs.journalbeat7; + extraConfig = '' + logging: + to_syslog: true + level: info + metrics.enabled: false + template.enabled: false + output.logstash: + hosts: [ "127.0.0.1:5044" ] + template.enabled: false + index: journalbeat + journalbeat.inputs: + - paths: [] + seek: cursor + ''; + }; + + services.logstash = { + enable = true; + # package = pkgs.logstash5; + # plugins = [ pkgs.logstash-contrib ]; + inputConfig = + '' + syslog { + timezone => "Etc/UTC" + } + beats { + port => 5044 + } + ''; + filterConfig = + '' + # Assume Beats + if [syslog] { + mutate { + add_field => { "program" => "%{[syslog][identifier]}" } + } + } + '' + + '' + if ![program] { + mutate { + add_field => { "program" => "unknown" } + } + } + '' + + '' + if ([program] == "logstash") { + drop {} + } + '' + + '' + if ( [program] == "dnsmasq") { + grok { + patterns_dir => ["${./patterns}"] + match => { + "message" => [ + "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype}\[[\w]+\] %{DOMAIN:domain} from %{IP}" + , "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype} %{DOMAIN:domain} is %{IPORWORD:resolved_ip}" + , "^%{DNSID:dnsid} %{IP:client}/%{PORT} %{DNSRESPONSE:dnstype} %{DOMAIN:domain} to %{IP:upstream_dns}" + ] + } + } + if [resolved_ip] { + geoip { + source => "resolved_ip" + } + } + mutate { + rename => { "host" => "syslog_host" } + } + # Target is to parse the the first and second significant part of the domain + grok { + patterns_dir => ["${./patterns}"] + match => { "domain" => [ "%{PUBLIC_SUFFIX:dns_suffix}$" ] } + } + if [client] { + mutate { copy => { "client" => "clientip" } } + dns { + reverse => [ "client"] + action => "replace" + hostsfile => [ "/etc/hosts" ] + hit_cache_ttl => 1600 + failed_cache_ttl => 60 + } + } + } + '' + '' + if ( [program] == "proftpd") { + kv { + field_split => " " + } + } + ''; + outputConfig = + '' + #stdout { + # codec => rubydebug + #} + elasticsearch { } + ''; + }; } |