From 726238d8717a03f6c80da31d4923632fed27e15a Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 1 Sep 2020 23:25:17 +0200 Subject: glados/wasser: giesszeit 10 -> 20 --- krebs/2configs/shack/glados/multi/wasser.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/glados/multi/wasser.nix b/krebs/2configs/shack/glados/multi/wasser.nix index 6f3dc98a..e3ba2838 100644 --- a/krebs/2configs/shack/glados/multi/wasser.nix +++ b/krebs/2configs/shack/glados/multi/wasser.nix @@ -2,7 +2,7 @@ # switch.crafting_giesskanne_relay let glados = import ../lib; - seconds = 10; + seconds = 20; wasser = "switch.crafting_giesskanne_relay"; in { -- cgit v1.2.3 From 440d1cc6642a22f4155fa616c5647cbd2bfbfb77 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 9 Sep 2020 00:32:10 +0200 Subject: shack/glados: re-enable influxdb output --- krebs/2configs/shack/glados/default.nix | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/glados/default.nix b/krebs/2configs/shack/glados/default.nix index 9bf90bca..e4f03295 100644 --- a/krebs/2configs/shack/glados/default.nix +++ b/krebs/2configs/shack/glados/default.nix @@ -62,13 +62,14 @@ in { ]; }; # https://www.home-assistant.io/components/influxdb/ - #influxdb = { - # database = "hass"; - # tags = { - # instance = "wolf"; - # source = "hass"; - # }; - #}; + influxdb = { + database = "glados"; + host = "influx.shack"; + tags = { + instance = "wolf"; + source = "glados"; + }; + }; esphome = {}; api = {}; mqtt = { -- cgit v1.2.3 From 306653db954b41759fccfbd8b9d42ff712fe64f0 Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 13 Sep 2020 13:56:58 +0200 Subject: wolf.r: more relevant nginx data --- krebs/2configs/shack/influx.nix | 5 +++++ krebs/2configs/shack/node-light.nix | 3 +++ 2 files changed, 8 insertions(+) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/influx.nix b/krebs/2configs/shack/influx.nix index 92cb24bf..93d83a59 100644 --- a/krebs/2configs/shack/influx.nix +++ b/krebs/2configs/shack/influx.nix @@ -8,6 +8,11 @@ in networking.firewall.allowedTCPPorts = [ port ]; # for legacy applications networking.firewall.allowedUDPPorts = [ collectd-port ]; services.nginx.virtualHosts."influx.shack" = { + # Disable constant GET request logging. + # $loggable map is defined in 1/wolf + extraConfig = '' + access_log syslog:server=unix:/dev/log combined if=$loggable; + ''; locations."/" = { proxyPass = "http://localhost:${toString port}/"; }; diff --git a/krebs/2configs/shack/node-light.nix b/krebs/2configs/shack/node-light.nix index b471f2af..4a981ea8 100644 --- a/krebs/2configs/shack/node-light.nix +++ b/krebs/2configs/shack/node-light.nix @@ -28,6 +28,9 @@ in { }; services.nginx.virtualHosts."openhab.shack" = { + extraConfig = '' + access_log syslog:server=unix:/dev/log combined if=$loggable; + ''; serverAliases = [ "lightapi.shack" ]; locations."/power/".proxyPass = "http://localhost:${port}/power/"; locations."/lounge/".proxyPass = "http://localhost:${port}/lounge/"; -- cgit v1.2.3 From eaf4ed0b24dcc89191d018fa4ebd9331e22ea324 Mon Sep 17 00:00:00 2001 From: makefu Date: Sun, 13 Sep 2020 23:52:22 +0200 Subject: shack/prometheus: activate alerting to telegram --- .../2configs/shack/prometheus/alertmanager-telegram.nix | 17 +++++++++++++++++ krebs/2configs/shack/prometheus/server.nix | 11 +++-------- 2 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 krebs/2configs/shack/prometheus/alertmanager-telegram.nix (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/prometheus/alertmanager-telegram.nix b/krebs/2configs/shack/prometheus/alertmanager-telegram.nix new file mode 100644 index 00000000..9d0ef45e --- /dev/null +++ b/krebs/2configs/shack/prometheus/alertmanager-telegram.nix @@ -0,0 +1,17 @@ +{ pkgs, ...}: +{ + systemd.services.alertmanager-bot-telegram = { + wantedBy = [ "multi-user.target" ]; + after = [ "ip-up.target" ]; + serviceConfig = { + EnvironmentFile = toString ; + DynamicUser = true; + StateDirectory = "alertbot"; + ExecStart = ''${pkgs.alertmanager-bot-telegram}/bin/alertmanager-bot \ + --alertmanager.url=http://alert.prometheus.shack --log.level=info \ + --store=bolt --bolt.path=/var/lib/alertbot/bot.db \ + --listen.addr="0.0.0.0:16320" \ + --template.paths=${pkgs.alertmanager-bot-telegram}/templates/default.tmpl''; + }; + }; +} diff --git a/krebs/2configs/shack/prometheus/server.nix b/krebs/2configs/shack/prometheus/server.nix index c088a3b0..8f37f447 100644 --- a/krebs/2configs/shack/prometheus/server.nix +++ b/krebs/2configs/shack/prometheus/server.nix @@ -118,7 +118,7 @@ ]; alertmanager = { enable = true; - listenAddress = "0.0.0.0"; + listenAddress = "127.0.0.1"; configuration = { "global" = { "smtp_smarthost" = "smtp.example.com:587"; @@ -134,15 +134,10 @@ "receivers" = [ { "name" = "team-admins"; - "email_configs" = [ - { - "to" = "devnull@example.com"; - "send_resolved" = true; - } - ]; + "email_configs" = [ ]; "webhook_configs" = [ { - "url" = "https://example.com/prometheus-alerts"; + "url" = "http://localhost:8080"; "send_resolved" = true; } ]; -- cgit v1.2.3 From 156339f63adcddcd8b1eb6d17bd4f76f72086920 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 14 Sep 2020 00:26:21 +0200 Subject: shack/prometheus: alertmanager-bot on a different port now --- krebs/2configs/shack/prometheus/alert-rules.nix | 5 ----- krebs/2configs/shack/prometheus/server.nix | 5 ++++- 2 files changed, 4 insertions(+), 6 deletions(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/prometheus/alert-rules.nix b/krebs/2configs/shack/prometheus/alert-rules.nix index 096c551b..afef5b8a 100644 --- a/krebs/2configs/shack/prometheus/alert-rules.nix +++ b/krebs/2configs/shack/prometheus/alert-rules.nix @@ -71,11 +71,6 @@ in mapAttrsToList (name: opts: { summary = "{{$labels.alias}}: Using more than 30% of its swap."; description = "{{$labels.alias}} is using 30% of its swap space for at least 30 minutes."; }; - node_visible_confluence_space = { - condition = "node_visible_confluence_space != 0"; - summary = "crowd prometheus cann see the {{$labels.space_name}} confluence space!"; - description = "crowd user `prometheus` can see the `{{$labels.space_name}}` confluence space."; - }; node_hwmon_temp = { condition = "node_hwmon_temp_celsius > node_hwmon_temp_crit_celsius*0.9 OR node_hwmon_temp_celsius > node_hwmon_temp_max_celsius*0.95"; time = "5m"; diff --git a/krebs/2configs/shack/prometheus/server.nix b/krebs/2configs/shack/prometheus/server.nix index 8f37f447..a10a43af 100644 --- a/krebs/2configs/shack/prometheus/server.nix +++ b/krebs/2configs/shack/prometheus/server.nix @@ -119,6 +119,9 @@ alertmanager = { enable = true; listenAddress = "127.0.0.1"; + webExternalUrl = "http://alert.prometheus.shack"; + logLevel = "debug"; + configuration = { "global" = { "smtp_smarthost" = "smtp.example.com:587"; @@ -137,7 +140,7 @@ "email_configs" = [ ]; "webhook_configs" = [ { - "url" = "http://localhost:8080"; + "url" = "http://localhost:16320"; "send_resolved" = true; } ]; -- cgit v1.2.3 From fc836fc0cb2aa2a9e65b2314ab361788d8e73186 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 14 Sep 2020 11:11:58 +0200 Subject: shack/prometheus: strip down number of alerts to 3 --- krebs/2configs/shack/prometheus/alert-rules.nix | 135 +++++++----------------- krebs/2configs/shack/prometheus/server.nix | 9 +- 2 files changed, 43 insertions(+), 101 deletions(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/prometheus/alert-rules.nix b/krebs/2configs/shack/prometheus/alert-rules.nix index afef5b8a..730921be 100644 --- a/krebs/2configs/shack/prometheus/alert-rules.nix +++ b/krebs/2configs/shack/prometheus/alert-rules.nix @@ -1,97 +1,42 @@ -{ lib }: -with lib; - +{ lib,... }: let - deviceFilter = ''device!="ramfs",device!="rpc_pipefs",device!="lxcfs",device!="nsfs",device!="borgfs"''; -in mapAttrsToList (name: opts: { - alert = name; - expr = opts.condition; - for = opts.time or "2m"; - labels = if (opts.page or true) then { severity = "page"; } else {}; - annotations = { - summary = opts.summary; - description = opts.description; - }; -}) { - node_down = { - condition = ''up{job="node"} == 0''; - summary = "{{$labels.alias}}: Node is down."; - description = "{{$labels.alias}} has been down for more than 2 minutes."; - }; - node_systemd_service_failed = { - condition = ''node_systemd_unit_state{state="failed"} == 1''; - summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start."; - description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}."; - }; - node_filesystem_full_80percent = { - condition = ''sort(node_filesystem_free_bytes{${deviceFilter}} < node_filesystem_size_bytes{${deviceFilter}} * 0.2) / 1024^3''; - time = "10m"; - summary = "{{$labels.alias}}: Filesystem is running out of space soon."; - description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 20% space left on its filesystem."; - }; - node_filesystem_full_in_7d = { - condition = ''predict_linear(node_filesystem_free_bytes{${deviceFilter}}[2d], 7*24*3600) <= 0''; - time = "1h"; - summary = "{{$labels.alias}}: Filesystem is running out of space in 7 days."; - description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 7 days"; - }; - node_filesystem_full_in_30d = { - condition = ''predict_linear(node_filesystem_free_bytes{${deviceFilter}}[30d], 30*24*3600) <= 0''; - time = "1h"; - summary = "{{$labels.alias}}: Filesystem is running out of space in 30 days."; - description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 30 days"; - }; - node_filedescriptors_full_in_3h = { - condition = ''predict_linear(node_filefd_allocated[3h], 3*3600) >= node_filefd_maximum''; - time = "20m"; - summary = "{{$labels.alias}} is running out of available file descriptors in 3 hours."; - description = "{{$labels.alias}} is running out of available file descriptors in approx. 3 hours"; - }; - node_filedescriptors_full_in_7d = { - condition = ''predict_linear(node_filefd_allocated[7d], 7*24*3600) >= node_filefd_maximum''; - time = "1h"; - summary = "{{$labels.alias}} is running out of available file descriptors in 7 days."; - description = "{{$labels.alias}} is running out of available file descriptors in approx. 7 days"; - }; - node_load15 = { - condition = ''node_load15 / on(alias) count(node_cpu_seconds_total{mode="system"}) by (alias) >= 1.0''; - time = "10m"; - summary = "{{$labels.alias}}: Running on high load: {{$value}}"; - description = "{{$labels.alias}} is running with load15 > 1 for at least 5 minutes: {{$value}}"; - }; - node_ram_using_90percent = { - condition = "node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1"; - time = "1h"; - summary = "{{$labels.alias}}: Using lots of RAM."; - description = "{{$labels.alias}} is using at least 90% of its RAM for at least 1 hour."; - }; - node_swap_using_30percent = { - condition = "node_memory_SwapTotal_bytes - (node_memory_SwapFree_bytes + node_memory_SwapCached_bytes) > node_memory_SwapTotal_bytes * 0.3"; - time = "30m"; - summary = "{{$labels.alias}}: Using more than 30% of its swap."; - description = "{{$labels.alias}} is using 30% of its swap space for at least 30 minutes."; - }; - node_hwmon_temp = { - condition = "node_hwmon_temp_celsius > node_hwmon_temp_crit_celsius*0.9 OR node_hwmon_temp_celsius > node_hwmon_temp_max_celsius*0.95"; - time = "5m"; - summary = "{{$labels.alias}}: Sensor {{$labels.sensor}}/{{$labels.chip}} temp is high: {{$value}} "; - description = "{{$labels.alias}} reports hwmon sensor {{$labels.sensor}}/{{$labels.chip}} temperature value is nearly critical: {{$value}}"; - }; - node_conntrack_limit = { - condition = "node_nf_conntrack_entries_limit - node_nf_conntrack_entries < 1000"; - time = "5m"; - summary = "{{$labels.alias}}: Number of tracked connections high"; - description = "{{$labels.alias}} has only {{$value}} free slots for connection tracking available."; - }; - node_reboot = { - condition = "time() - node_boot_time_seconds < 300"; - summary = "{{$labels.alias}}: Reboot"; - description = "{{$labels.alias}} just rebooted."; - }; - node_uptime = { - condition = "time() - node_boot_time_seconds > 2592000"; - page = false; - summary = "{{$labels.alias}}: Uptime monster"; - description = "{{$labels.alias}} has been up for more than 30 days."; - }; + disk_free_threshold = "10"; # at least this much free disk percentage +in { + services.prometheus.rules = [(builtins.toJSON + { + groups = [ + { name = "shack-env"; + rules = [ + { + alert = "RootPartitionFull"; + for = "30m"; + expr = ''(node_filesystem_avail_bytes{alias="wolf",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="wolf",mountpoint="/"} < ${disk_free_threshold}''; + labels.severity = "warning"; + annotations.summary = "{{ $labels.alias }} root disk full"; + annotations.url = "http://grafana.shack/"; + annotations.description = ''The root disk of {{ $labels.alias }} has {{ $value | printf "%.2f" }}% free disk space (Threshold at ${disk_free_threshold}%).A vast number of shackspace services will stop working. CI for deploying new configuration will also seize working. Log in to the system and run `nix-collect-garbage -d` and clean up the shack share folder in `/home/share` .If this does not help you can check `du -hs /var/ | sort -h`, run `docker system prune` or if you are really desperate run `du -hs / | sort -h` and go through the folders recursively until you've found something to delete''; + } + { + alert = "RootPartitionFull"; + for = "30m"; + expr = ''(node_filesystem_avail_bytes{alias="puyak",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="puyak",mountpoint="/"} < ${disk_free_threshold}''; + labels.severity = "warning"; + annotations.summary = "{{ $labels.alias }} root disk full"; + annotations.url = "http://grafana.shack/"; + annotations.description = ''The root disk of {{ $labels.alias }} has {{ $value | printf "%.2f" }}% free disk space (Threshold at ${disk_free_threshold}%).Prometheus will not be able to create new alerts and CI for deploying new configuration will also seize working. Log in to the system and run `nix-collect-garbage -d` and if this does not help you can check `du -hs /var/ | sort -h`, run `docker system prune` or if you are really desperate run `du -hs / | sort -h` and go through the folders recursively until you've found something to delete''; + } + { + alert = "HostDown"; + expr = ''up{alias="wolf"} == 0''; + for = "5m"; + labels.severity = "page"; + annotations.summary = "Instance {{ $labels.alias }} down for 5 minutes"; + annotations.url = "http://grafana.shack/"; + annotations.description = ''Host {{ $labels.alias }} went down and has not been reconnected after 5 minutes. This is probably bad news, try to restart the host via naproxen ( http://naproxen.shack:8006 ). Wolf being down means that CI,glados automation, light management and a couple of other services will not work anymore.''; + } + ]; + } + ]; + } + )]; } diff --git a/krebs/2configs/shack/prometheus/server.nix b/krebs/2configs/shack/prometheus/server.nix index a10a43af..9e4b4d1a 100644 --- a/krebs/2configs/shack/prometheus/server.nix +++ b/krebs/2configs/shack/prometheus/server.nix @@ -1,6 +1,9 @@ { pkgs, lib, config, ... }: # from https://gist.github.com/globin/02496fd10a96a36f092a8e7ea0e6c7dd { + imports = [ + ./alert-rules.nix + ]; networking = { firewall.allowedTCPPorts = [ 9090 # prometheus @@ -18,12 +21,6 @@ }; prometheus = { enable = true; - ruleFiles = lib.singleton (pkgs.writeText "prometheus-rules.yml" (builtins.toJSON { - groups = lib.singleton { - name = "mf-alerting-rules"; - rules = import ./alert-rules.nix { inherit lib; }; - }; - })); scrapeConfigs = [ { job_name = "node"; -- cgit v1.2.3 From df4f3982b15bf107db1b2c5cad58167e30fb3dec Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 14 Sep 2020 11:42:32 +0200 Subject: shack/prometheus: use correct alias for rules --- krebs/2configs/shack/prometheus/alert-rules.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/prometheus/alert-rules.nix b/krebs/2configs/shack/prometheus/alert-rules.nix index 730921be..0da1128a 100644 --- a/krebs/2configs/shack/prometheus/alert-rules.nix +++ b/krebs/2configs/shack/prometheus/alert-rules.nix @@ -10,7 +10,7 @@ in { { alert = "RootPartitionFull"; for = "30m"; - expr = ''(node_filesystem_avail_bytes{alias="wolf",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="wolf",mountpoint="/"} < ${disk_free_threshold}''; + expr = ''(node_filesystem_avail_bytes{alias="wolf.shack",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="wolf.shack",mountpoint="/"} < ${disk_free_threshold}''; labels.severity = "warning"; annotations.summary = "{{ $labels.alias }} root disk full"; annotations.url = "http://grafana.shack/"; @@ -19,7 +19,7 @@ in { { alert = "RootPartitionFull"; for = "30m"; - expr = ''(node_filesystem_avail_bytes{alias="puyak",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="puyak",mountpoint="/"} < ${disk_free_threshold}''; + expr = ''(node_filesystem_avail_bytes{alias="puyak.shack",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="puyak.shack",mountpoint="/"} < ${disk_free_threshold}''; labels.severity = "warning"; annotations.summary = "{{ $labels.alias }} root disk full"; annotations.url = "http://grafana.shack/"; @@ -27,7 +27,7 @@ in { } { alert = "HostDown"; - expr = ''up{alias="wolf"} == 0''; + expr = ''up{alias="wolf.shack"} == 0''; for = "5m"; labels.severity = "page"; annotations.summary = "Instance {{ $labels.alias }} down for 5 minutes"; -- cgit v1.2.3 From 88aee4087ebf00e32b39476ddbd82c7e69490280 Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 14 Sep 2020 15:45:37 +0200 Subject: shack/prometheus: add urls to alerts --- krebs/2configs/shack/prometheus/alert-rules.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/prometheus/alert-rules.nix b/krebs/2configs/shack/prometheus/alert-rules.nix index 0da1128a..1c2d0b1a 100644 --- a/krebs/2configs/shack/prometheus/alert-rules.nix +++ b/krebs/2configs/shack/prometheus/alert-rules.nix @@ -13,7 +13,7 @@ in { expr = ''(node_filesystem_avail_bytes{alias="wolf.shack",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="wolf.shack",mountpoint="/"} < ${disk_free_threshold}''; labels.severity = "warning"; annotations.summary = "{{ $labels.alias }} root disk full"; - annotations.url = "http://grafana.shack/"; + annotations.url = "http://grafana.shack/d/hb7fSE0Zz/shack-system-dashboard?orgId=1&var-job=node&var-hostname=All&var-node=wolf.shack:9100&var-device=All&var-maxmount=%2F&var-show_hostname=wolf"; annotations.description = ''The root disk of {{ $labels.alias }} has {{ $value | printf "%.2f" }}% free disk space (Threshold at ${disk_free_threshold}%).A vast number of shackspace services will stop working. CI for deploying new configuration will also seize working. Log in to the system and run `nix-collect-garbage -d` and clean up the shack share folder in `/home/share` .If this does not help you can check `du -hs /var/ | sort -h`, run `docker system prune` or if you are really desperate run `du -hs / | sort -h` and go through the folders recursively until you've found something to delete''; } { @@ -22,7 +22,7 @@ in { expr = ''(node_filesystem_avail_bytes{alias="puyak.shack",mountpoint="/"} * 100) / node_filesystem_size_bytes{alias="puyak.shack",mountpoint="/"} < ${disk_free_threshold}''; labels.severity = "warning"; annotations.summary = "{{ $labels.alias }} root disk full"; - annotations.url = "http://grafana.shack/"; + annotations.url = "http://grafana.shack/d/hb7fSE0Zz/shack-system-dashboard?orgId=1&var-job=node&var-hostname=All&var-node=wolf.shack:9100&var-device=All&var-maxmount=%2F&var-show_hostname=puyak"; annotations.description = ''The root disk of {{ $labels.alias }} has {{ $value | printf "%.2f" }}% free disk space (Threshold at ${disk_free_threshold}%).Prometheus will not be able to create new alerts and CI for deploying new configuration will also seize working. Log in to the system and run `nix-collect-garbage -d` and if this does not help you can check `du -hs /var/ | sort -h`, run `docker system prune` or if you are really desperate run `du -hs / | sort -h` and go through the folders recursively until you've found something to delete''; } { @@ -31,7 +31,7 @@ in { for = "5m"; labels.severity = "page"; annotations.summary = "Instance {{ $labels.alias }} down for 5 minutes"; - annotations.url = "http://grafana.shack/"; + annotations.url = "http://grafana.shack/d/hb7fSE0Zz/shack-system-dashboard?orgId=1&var-job=node&var-hostname=All&var-node=wolf.shack:9100&var-device=All&var-maxmount=%2F&var-show_hostname=wolf"; annotations.description = ''Host {{ $labels.alias }} went down and has not been reconnected after 5 minutes. This is probably bad news, try to restart the host via naproxen ( http://naproxen.shack:8006 ). Wolf being down means that CI,glados automation, light management and a couple of other services will not work anymore.''; } ]; -- cgit v1.2.3 From 1394ae1bd4bc3dcfb77b0daee996e75f8092fc5b Mon Sep 17 00:00:00 2001 From: makefu Date: Mon, 14 Sep 2020 16:56:30 +0200 Subject: shack/prometheus: add custom shack template --- .../shack/prometheus/alertmanager-telegram.nix | 4 ++-- .../2configs/shack/prometheus/templates/shack.tmpl | 25 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 krebs/2configs/shack/prometheus/templates/shack.tmpl (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/prometheus/alertmanager-telegram.nix b/krebs/2configs/shack/prometheus/alertmanager-telegram.nix index 9d0ef45e..2df63e52 100644 --- a/krebs/2configs/shack/prometheus/alertmanager-telegram.nix +++ b/krebs/2configs/shack/prometheus/alertmanager-telegram.nix @@ -8,10 +8,10 @@ DynamicUser = true; StateDirectory = "alertbot"; ExecStart = ''${pkgs.alertmanager-bot-telegram}/bin/alertmanager-bot \ - --alertmanager.url=http://alert.prometheus.shack --log.level=info \ + --alertmanager.url=http://alert.prometheus.shack --log.level=debug \ --store=bolt --bolt.path=/var/lib/alertbot/bot.db \ --listen.addr="0.0.0.0:16320" \ - --template.paths=${pkgs.alertmanager-bot-telegram}/templates/default.tmpl''; + --template.paths=${./templates}/shack.tmpl''; }; }; } diff --git a/krebs/2configs/shack/prometheus/templates/shack.tmpl b/krebs/2configs/shack/prometheus/templates/shack.tmpl new file mode 100644 index 00000000..9295f019 --- /dev/null +++ b/krebs/2configs/shack/prometheus/templates/shack.tmpl @@ -0,0 +1,25 @@ +{{ define "telegram.default" }} +{{range .Alerts -}} +{{ $severity := index .Labels "severity" }} +{{ $desc := "No Description" }} +{{ if eq .Status "firing" }} + {{ $desc = index .Annotations "description" }} + {{- if eq $severity "critical" -}} + [CRITICAL] + {{- else if eq $severity "warning" -}} + [WARNING] + {{- else -}} + [{{ $severity }}] + {{- end -}} +{{ else -}} + {{ $desc = "The issue has been resolved" }} + [RESOLVED] +{{- end }} {{ index .Labels "alertname"}}: {{ index .Annotations "summary"}} + +{{ $desc }} + +Alert Links: +* Grafana +* Source +{{end -}} +{{end}} -- cgit v1.2.3 From 91b7a2a58490a59d6090870a9db9eca8d11d9ab4 Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 15 Sep 2020 20:16:10 +0200 Subject: go: return go.r for shortened urls --- krebs/2configs/go.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/go.nix b/krebs/2configs/go.nix index f4c1290c..c39b08a8 100644 --- a/krebs/2configs/go.nix +++ b/krebs/2configs/go.nix @@ -13,7 +13,7 @@ with import ; enable = true; virtualHosts.go = { locations."/".extraConfig = '' - proxy_set_header Host go; + proxy_set_header Host go.r; proxy_pass http://localhost:1337; ''; serverAliases = [ -- cgit v1.2.3 From e6a8d378460827bab9a0fd45a50faa5eca7e8ae0 Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 15 Sep 2020 22:52:34 +0200 Subject: reaktor2: nicks can have special characters --- krebs/2configs/reaktor2.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/reaktor2.nix b/krebs/2configs/reaktor2.nix index 72eff176..061dc9ab 100644 --- a/krebs/2configs/reaktor2.nix +++ b/krebs/2configs/reaktor2.nix @@ -76,7 +76,7 @@ let }; } { - pattern = ''^([\w-]*):?\s+([+-][1-9][0-9]*)\s+(\S+)$''; + pattern = ''^([\H-]*):?\s+([+-][1-9][0-9]*)\s+(\S+)$''; activate = "match"; arguments = [1 2 3]; command = { -- cgit v1.2.3 From 8ed22f337c76df2ad592650ece7abe1a414c4182 Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 16 Sep 2020 22:21:53 +0200 Subject: shack/glados: implement rollo triggers --- krebs/2configs/shack/glados/default.nix | 10 ++++-- krebs/2configs/shack/glados/multi/rollos.nix | 53 +++++++++++++++++++++++++--- krebs/2configs/shack/glados/multi/wasser.nix | 10 ++++-- 3 files changed, 63 insertions(+), 10 deletions(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/glados/default.nix b/krebs/2configs/shack/glados/default.nix index e4f03295..f47bca2d 100644 --- a/krebs/2configs/shack/glados/default.nix +++ b/krebs/2configs/shack/glados/default.nix @@ -3,6 +3,7 @@ let shackopen = import ./multi/shackopen.nix; wasser = import ./multi/wasser.nix; badair = import ./multi/schlechte_luft.nix; + rollos = import ./multi/rollos.nix; in { services.nginx.virtualHosts."hass.shack" = { serverAliases = [ "glados.shack" ]; @@ -65,6 +66,10 @@ in { influxdb = { database = "glados"; host = "influx.shack"; + component_config_glob = { + "sensor.*particulate_matter_2_5um_concentration".override_measurement = "2_5um particles"; + "sensor.*particulate_matter_10_0um_concentration".override_measurement ="10um particles"; + }; tags = { instance = "wolf"; source = "glados"; @@ -94,8 +99,7 @@ in { }; }; switch = - wasser.switch - ++ (import ./switch/power.nix) + (import ./switch/power.nix) ; light = []; media_player = [ @@ -114,6 +118,7 @@ in { ++ (import ./sensors/mate.nix) ++ (import ./sensors/darksky.nix { inherit lib;}) ++ shackopen.sensor + ++ wasser.sensor ; air_quality = (import ./sensors/sensemap.nix ); @@ -148,6 +153,7 @@ in { automation = wasser.automation ++ badair.automation + ++ rollos.automation ++ (import ./automation/shack-startup.nix) ++ (import ./automation/party-time.nix) ++ (import ./automation/hass-restart.nix); diff --git a/krebs/2configs/shack/glados/multi/rollos.nix b/krebs/2configs/shack/glados/multi/rollos.nix index 1febad52..4e649493 100644 --- a/krebs/2configs/shack/glados/multi/rollos.nix +++ b/krebs/2configs/shack/glados/multi/rollos.nix @@ -1,13 +1,56 @@ +# + let glados = import ../lib; + tempsensor = "sensor.dark_sky_temperature"; + all_covers = [ + "cover.crafting_rollo" + "cover.elab_rollo" + "cover.or2_rollo" + "cover.retroraum_rollo" + ]; in { - # LED - light = [ - ]; - sensor = [ - ]; automation = [ + { alias = "Rollos fahren Runter"; + trigger = [ + { + platform = "numeric_state"; + entity_id = tempsensor; + above = 25; + for = "00:30:00"; + } + ]; + condition = + [ + { + condition = "state"; + entity_id = "sun.sun"; + state = "above_horizon"; + } + ]; + action = + [ + { service = "cover.close_cover"; + entity_id = all_covers; + } + ]; + } + { alias = "Rollos fahren Hoch"; + trigger = [ + { + platform = "sun"; + event = "sunset"; + } + ]; + condition = [ ]; + action = + [ + { service = "cover.open_cover"; + entity_id = all_covers; + } + ]; + } ]; } diff --git a/krebs/2configs/shack/glados/multi/wasser.nix b/krebs/2configs/shack/glados/multi/wasser.nix index e3ba2838..74ce736a 100644 --- a/krebs/2configs/shack/glados/multi/wasser.nix +++ b/krebs/2configs/shack/glados/multi/wasser.nix @@ -6,9 +6,13 @@ let wasser = "switch.crafting_giesskanne_relay"; in { - switch = [ - (glados.tasmota.plug { host = "Wasser"; topic = "plug";} ) - ]; + sensor = map ( entity_id: { + platform = "statistics"; + name = "Statistics for ${entity_id}"; + inherit entity_id; + max_age.minutes = "60"; + }) ["sensor.crafting_brotbox_soil_moisture"]; + automation = [ -- cgit v1.2.3 From a0c494db8b26f2b140116bc29df4e42c206e86cd Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 18 Sep 2020 10:59:05 +0200 Subject: shack/muell_mail: bump revision --- krebs/2configs/shack/muell_mail.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/muell_mail.nix b/krebs/2configs/shack/muell_mail.nix index 40927895..48156471 100644 --- a/krebs/2configs/shack/muell_mail.nix +++ b/krebs/2configs/shack/muell_mail.nix @@ -4,8 +4,8 @@ let pkg = pkgs.callPackage ( pkgs.fetchgit { url = "https://git.shackspace.de/rz/muell_mail"; - rev = "57b67c95052d90044137b2c89007a371dc389afd"; - sha256 = "1grkzs6fxjnc2bv4kskj63d5sb4qxz6yyr85nj0da9hn7qkk4jkj"; + rev = "c3e43687879f95e01a82ef176fa15678543b2eb8"; + sha256 = "0hgchwam5ma96s2v6mx2jfkh833psadmisjbm3k3153rlxp46frx"; }) { mkYarnPackage = pkgs.yarn2nix-moretea.mkYarnPackage; }; home = "/var/lib/muell_mail"; cfg = toString ; -- cgit v1.2.3 From 9ddea79d0537eddee1e6d372403bc2f3bbe07675 Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Sep 2020 00:53:54 +0200 Subject: shack/prometheus: set loglevel to info --- krebs/2configs/shack/prometheus/alertmanager-telegram.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/prometheus/alertmanager-telegram.nix b/krebs/2configs/shack/prometheus/alertmanager-telegram.nix index 2df63e52..8527001c 100644 --- a/krebs/2configs/shack/prometheus/alertmanager-telegram.nix +++ b/krebs/2configs/shack/prometheus/alertmanager-telegram.nix @@ -8,7 +8,7 @@ DynamicUser = true; StateDirectory = "alertbot"; ExecStart = ''${pkgs.alertmanager-bot-telegram}/bin/alertmanager-bot \ - --alertmanager.url=http://alert.prometheus.shack --log.level=debug \ + --alertmanager.url=http://alert.prometheus.shack --log.level=info \ --store=bolt --bolt.path=/var/lib/alertbot/bot.db \ --listen.addr="0.0.0.0:16320" \ --template.paths=${./templates}/shack.tmpl''; -- cgit v1.2.3 From 49df24b5bbfd61b7d86f65d222fc0534732cbc88 Mon Sep 17 00:00:00 2001 From: makefu Date: Tue, 22 Sep 2020 00:54:25 +0200 Subject: wolf.r: manage light.shack web-interface in shackspace --- krebs/2configs/shack/light.shack.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 krebs/2configs/shack/light.shack.nix (limited to 'krebs/2configs') diff --git a/krebs/2configs/shack/light.shack.nix b/krebs/2configs/shack/light.shack.nix new file mode 100644 index 00000000..8e01cb1b --- /dev/null +++ b/krebs/2configs/shack/light.shack.nix @@ -0,0 +1,12 @@ +{ config, pkgs, ... }: +let + light-shack-src = pkgs.fetchgit { + url = "https://git.shackspace.de/rz/standby.shack"; + rev = "e1b90a0a"; + sha256 = "07fmz63arc5rxa0a3778srwz0jflp4ad6xnwkkc56hwybby0bclh"; + }; + web-dir = "${light-shack-src}/client/www/"; +in +{ + services.nginx.virtualHosts."light.shack".locations."/".root = web-dir; +} -- cgit v1.2.3