summaryrefslogtreecommitdiffstats
path: root/lass/2configs
diff options
context:
space:
mode:
Diffstat (limited to 'lass/2configs')
-rw-r--r--lass/2configs/AP.nix77
-rw-r--r--lass/2configs/backup.nix20
-rw-r--r--lass/2configs/backups.nix173
-rw-r--r--lass/2configs/baseX.nix1
-rw-r--r--lass/2configs/bitcoin.nix10
-rw-r--r--lass/2configs/dcso-dev.nix1
-rw-r--r--lass/2configs/default.nix3
-rw-r--r--lass/2configs/dns-stuff.nix16
-rw-r--r--lass/2configs/exim-smarthost.nix1
-rw-r--r--lass/2configs/gc.nix2
-rw-r--r--lass/2configs/git.nix14
-rw-r--r--lass/2configs/go.nix19
-rw-r--r--lass/2configs/mail.nix5
-rw-r--r--lass/2configs/monitoring/client.nix26
-rw-r--r--lass/2configs/monitoring/monit-alarms.nix44
-rw-r--r--lass/2configs/monitoring/node-exporter.nix15
-rw-r--r--lass/2configs/monitoring/prometheus-server.nix217
-rw-r--r--lass/2configs/monitoring/server.nix87
-rw-r--r--lass/2configs/reaktor-coders.nix15
-rw-r--r--lass/2configs/repo-sync.nix1
-rw-r--r--lass/2configs/syncthing.nix1
-rw-r--r--lass/2configs/websites/lassulus.nix56
-rw-r--r--lass/2configs/websites/util.nix59
-rw-r--r--lass/2configs/zsh.nix4
24 files changed, 440 insertions, 427 deletions
diff --git a/lass/2configs/AP.nix b/lass/2configs/AP.nix
new file mode 100644
index 000000000..5ce7cfff8
--- /dev/null
+++ b/lass/2configs/AP.nix
@@ -0,0 +1,77 @@
+{ config, pkgs, ... }:
+with import <stockholm/lib>;
+let
+ wifi = "wlp0s29u1u2";
+in {
+ boot.extraModulePackages = [
+ pkgs.linuxPackages.rtl8814au
+ ];
+ networking.networkmanager.unmanaged = [ wifi ];
+
+ systemd.services.hostapd = {
+ description = "hostapd wireless AP";
+ path = [ pkgs.hostapd ];
+ wantedBy = [ "network.target" ];
+
+ after = [ "${wifi}-cfg.service" "nat.service" "bind.service" "dhcpd.service" "sys-subsystem-net-devices-${wifi}.device" ];
+
+ serviceConfig = {
+ ExecStart = "${pkgs.hostapd}/bin/hostapd ${pkgs.writeText "hostapd.conf" ''
+ interface=${wifi}
+ hw_mode=a
+ channel=36
+ ieee80211d=1
+ country_code=DE
+ ieee80211n=1
+ ieee80211ac=1
+ wmm_enabled=1
+
+ # 5ghz
+ ssid=krebsing
+ auth_algs=1
+ wpa=2
+ wpa_key_mgmt=WPA-PSK
+ rsn_pairwise=CCMP
+ wpa_passphrase=aidsballz
+ ''}";
+ Restart = "always";
+ };
+ };
+
+ networking.interfaces.${wifi}.ipv4.addresses = [
+ { address = "10.99.0.1"; prefixLength = 24; }
+ ];
+ services.dhcpd4 = {
+ enable = true;
+ interfaces = [ wifi ];
+ extraConfig = ''
+ option subnet-mask 255.255.255.0;
+ option routers 10.99.0.1;
+ option domain-name-servers 1.1.1.1, 8.8.8.8;
+ subnet 10.99.0.0 netmask 255.255.255.0 {
+ range 10.99.0.100 10.99.0.200;
+ }
+ '';
+ };
+
+ boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
+ krebs.iptables.tables.filter.FORWARD.rules = [
+ { v6 = false; predicate = "-d 10.99.0.0/24 -o ${wifi} -m conntrack --ctstate RELATED,ESTABLISHED"; target = "ACCEPT"; }
+ { v6 = false; predicate = "-s 10.99.0.0/24 -i ${wifi}"; target = "ACCEPT"; }
+ { v6 = false; predicate = "-i ${wifi} -o ${wifi}"; target = "ACCEPT"; }
+ { v6 = false; predicate = "-o ${wifi}"; target = "REJECT --reject-with icmp-port-unreachable"; }
+ { v6 = false; predicate = "-i ${wifi}"; target = "REJECT --reject-with icmp-port-unreachable"; }
+ ];
+ krebs.iptables.tables.nat.PREROUTING.rules = [
+ { v6 = false; predicate = "-s 10.99.0.0/24"; target = "ACCEPT"; precedence = 1000; }
+ ];
+ krebs.iptables.tables.nat.POSTROUTING.rules = [
+ #TODO find out what this is about?
+ { v6 = false; predicate = "-s 10.99.0.0/24 -d 224.0.0.0/24"; target = "RETURN"; }
+ { v6 = false; predicate = "-s 10.99.0.0/24 -d 255.255.255.255"; target = "RETURN"; }
+
+ { v6 = false; predicate = "-s 10.99.0.0/24 ! -d 10.99.0.0/24"; target = "MASQUERADE"; }
+ { v6 = false; predicate = "-s 10.99.0.0/24 ! -d 10.99.0.0/24 -p tcp"; target = "MASQUERADE --to-ports 1024-65535"; }
+ { v6 = false; predicate = "-s 10.99.0.0/24 ! -d 10.99.0.0/24 -p udp"; target = "MASQUERADE --to-ports 1024-65535"; }
+ ];
+}
diff --git a/lass/2configs/backup.nix b/lass/2configs/backup.nix
new file mode 100644
index 000000000..27adf6d2a
--- /dev/null
+++ b/lass/2configs/backup.nix
@@ -0,0 +1,20 @@
+{ config, lib, ... }:
+with import <stockholm/lib>;
+
+{
+ fileSystems = {
+ "/backups" = {
+ device = "/dev/pool/backup";
+ fsType = "ext4";
+ };
+ };
+ users.users.backup = {
+ useDefaultShell = true;
+ home = "/backups";
+ createHome = true;
+ openssh.authorizedKeys.keys = with config.krebs.hosts; [
+ mors.ssh.pubkey
+ prism.ssh.pubkey
+ ];
+ };
+}
diff --git a/lass/2configs/backups.nix b/lass/2configs/backups.nix
deleted file mode 100644
index c4fb85420..000000000
--- a/lass/2configs/backups.nix
+++ /dev/null
@@ -1,173 +0,0 @@
-{ config, lib, ... }:
-with import <stockholm/lib>;
-{
-
- # TODO add timerConfig to krebs.backup and randomize startup
- # TODO define plans more abstract
- krebs.backup.plans = {
- } // mapAttrs (_: recursiveUpdate {
- snapshots = {
- daily = { format = "%Y-%m-%d"; retain = 7; };
- weekly = { format = "%YW%W"; retain = 4; };
- monthly = { format = "%Y-%m"; retain = 12; };
- yearly = { format = "%Y"; };
- };
- }) {
- dishfire-http-prism = {
- method = "pull";
- src = { host = config.krebs.hosts.dishfire; path = "/srv/http"; };
- dst = { host = config.krebs.hosts.prism; path = "/bku/dishfire-http"; };
- startAt = "03:00";
- };
- dishfire-http-icarus = {
- method = "pull";
- src = { host = config.krebs.hosts.dishfire; path = "/srv/http"; };
- dst = { host = config.krebs.hosts.icarus; path = "/bku/dishfire-http"; };
- startAt = "03:10";
- };
- dishfire-http-mors = {
- method = "pull";
- src = { host = config.krebs.hosts.dishfire; path = "/srv/http"; };
- dst = { host = config.krebs.hosts.mors; path = "/bku/dishfire-http"; };
- startAt = "03:05";
- };
- dishfire-http-shodan = {
- method = "pull";
- src = { host = config.krebs.hosts.dishfire; path = "/srv/http"; };
- dst = { host = config.krebs.hosts.shodan; path = "/bku/dishfire-http"; };
- startAt = "03:10";
- };
- dishfire-sql-prism = {
- method = "pull";
- src = { host = config.krebs.hosts.dishfire; path = "/bku/sql_dumps"; };
- dst = { host = config.krebs.hosts.prism; path = "/bku/dishfire-sql"; };
- startAt = "03:15";
- };
- dishfire-sql-icarus = {
- method = "pull";
- src = { host = config.krebs.hosts.dishfire; path = "/bku/sql_dumps"; };
- dst = { host = config.krebs.hosts.icarus; path = "/bku/dishfire-sql"; };
- startAt = "03:25";
- };
- dishfire-sql-mors = {
- method = "pull";
- src = { host = config.krebs.hosts.dishfire; path = "/bku/sql_dumps"; };
- dst = { host = config.krebs.hosts.mors; path = "/bku/dishfire-sql"; };
- startAt = "03:20";
- };
- dishfire-sql-shodan = {
- method = "pull";
- src = { host = config.krebs.hosts.dishfire; path = "/bku/sql_dumps"; };
- dst = { host = config.krebs.hosts.shodan; path = "/bku/dishfire-sql"; };
- startAt = "03:25";
- };
- prism-bitlbee-icarus = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/var/lib/bitlbee"; };
- dst = { host = config.krebs.hosts.icarus; path = "/bku/prism-bitlbee"; };
- startAt = "03:25";
- };
- prism-bitlbee-mors = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/var/lib/bitlbee"; };
- dst = { host = config.krebs.hosts.mors; path = "/bku/prism-bitlbee"; };
- startAt = "03:25";
- };
- prism-bitlbee-shodan = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/var/lib/bitlbee"; };
- dst = { host = config.krebs.hosts.shodan; path = "/bku/prism-bitlbee"; };
- startAt = "03:25";
- };
- prism-chat-icarus = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/home/chat"; };
- dst = { host = config.krebs.hosts.icarus; path = "/bku/prism-chat"; };
- startAt = "03:35";
- };
- prism-chat-mors = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/home/chat"; };
- dst = { host = config.krebs.hosts.mors; path = "/bku/prism-chat"; };
- startAt = "03:30";
- };
- prism-chat-shodan = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/home/chat"; };
- dst = { host = config.krebs.hosts.shodan; path = "/bku/prism-chat"; };
- startAt = "03:35";
- };
- prism-sql-icarus = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/bku/sql_dumps"; };
- dst = { host = config.krebs.hosts.icarus; path = "/bku/prism-sql_dumps"; };
- startAt = "03:45";
- };
- prism-sql-mors = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/bku/sql_dumps"; };
- dst = { host = config.krebs.hosts.mors; path = "/bku/prism-sql_dumps"; };
- startAt = "03:40";
- };
- prism-sql-shodan = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/bku/sql_dumps"; };
- dst = { host = config.krebs.hosts.shodan; path = "/bku/prism-sql_dumps"; };
- startAt = "03:45";
- };
- prism-http-icarus = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/srv/http"; };
- dst = { host = config.krebs.hosts.icarus; path = "/bku/prism-http"; };
- startAt = "03:55";
- };
- prism-http-mors = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/srv/http"; };
- dst = { host = config.krebs.hosts.mors; path = "/bku/prism-http"; };
- startAt = "03:50";
- };
- prism-http-shodan = {
- method = "pull";
- src = { host = config.krebs.hosts.prism; path = "/srv/http"; };
- dst = { host = config.krebs.hosts.shodan; path = "/bku/prism-http"; };
- startAt = "03:55";
- };
- icarus-home-mors = {
- method = "pull";
- src = { host = config.krebs.hosts.icarus; path = "/home"; };
- dst = { host = config.krebs.hosts.mors; path = "/bku/icarus-home"; };
- startAt = "05:00";
- };
- icarus-home-shodan = {
- method = "push";
- src = { host = config.krebs.hosts.icarus; path = "/home"; };
- dst = { host = config.krebs.hosts.shodan; path = "/bku/icarus-home"; };
- startAt = "05:00";
- };
- mors-home-icarus = {
- method = "push";
- src = { host = config.krebs.hosts.mors; path = "/home"; };
- dst = { host = config.krebs.hosts.icarus; path = "/bku/mors-home"; };
- startAt = "05:00";
- };
- mors-home-shodan = {
- method = "push";
- src = { host = config.krebs.hosts.mors; path = "/home"; };
- dst = { host = config.krebs.hosts.shodan; path = "/bku/mors-home"; };
- startAt = "05:00";
- };
- shodan-home-icarus = {
- method = "pull";
- src = { host = config.krebs.hosts.shodan; path = "/home"; };
- dst = { host = config.krebs.hosts.icarus; path = "/bku/shodan-home"; };
- startAt = "04:00";
- };
- shodan-home-mors = {
- method = "pull";
- src = { host = config.krebs.hosts.shodan; path = "/home"; };
- dst = { host = config.krebs.hosts.mors; path = "/bku/shodan-home"; };
- startAt = "04:00";
- };
- };
-}
diff --git a/lass/2configs/baseX.nix b/lass/2configs/baseX.nix
index ed179ded6..e2e44b6fc 100644
--- a/lass/2configs/baseX.nix
+++ b/lass/2configs/baseX.nix
@@ -9,7 +9,6 @@ in {
./power-action.nix
./copyq.nix
./livestream.nix
- ./dns-stuff.nix
./urxvt.nix
./network-manager.nix
{
diff --git a/lass/2configs/bitcoin.nix b/lass/2configs/bitcoin.nix
index a405addfc..9f6fd3bf0 100644
--- a/lass/2configs/bitcoin.nix
+++ b/lass/2configs/bitcoin.nix
@@ -10,9 +10,6 @@ in {
krebs.per-user.bitcoin.packages = [
pkgs.electrum
];
- krebs.per-user.ethereum.packages = [
- pkgs.go-ethereum
- ];
users.extraUsers = {
bch = {
name = "bch";
@@ -28,13 +25,6 @@ in {
useDefaultShell = true;
createHome = true;
};
- ethereum = {
- name = "ethereum";
- description = "user for ethereum stuff";
- home = "/home/ethereum";
- useDefaultShell = true;
- createHome = true;
- };
};
security.sudo.extraConfig = ''
${mainUser.name} ALL=(bitcoin) NOPASSWD: ALL
diff --git a/lass/2configs/dcso-dev.nix b/lass/2configs/dcso-dev.nix
index ae1c7bc8d..b985b67b3 100644
--- a/lass/2configs/dcso-dev.nix
+++ b/lass/2configs/dcso-dev.nix
@@ -9,6 +9,7 @@ in {
dev = {
name = "dev";
uid = genid "dev";
+ extraGroups = [ "docker" ];
description = "user for collaborative development";
home = "/home/dev";
useDefaultShell = true;
diff --git a/lass/2configs/default.nix b/lass/2configs/default.nix
index 5a5f1b347..12a814605 100644
--- a/lass/2configs/default.nix
+++ b/lass/2configs/default.nix
@@ -6,10 +6,9 @@ with import <stockholm/lib>;
./gc.nix
./mc.nix
./vim.nix
- ./monitoring/client.nix
+ ./monitoring/node-exporter.nix
./zsh.nix
./htop.nix
- ./backups.nix
./security-workarounds.nix
{
users.extraUsers =
diff --git a/lass/2configs/dns-stuff.nix b/lass/2configs/dns-stuff.nix
deleted file mode 100644
index cbcce8df9..000000000
--- a/lass/2configs/dns-stuff.nix
+++ /dev/null
@@ -1,16 +0,0 @@
-{ config, pkgs, ... }:
-with import <stockholm/lib>;
-{
- services.dnscrypt-proxy = {
- enable = true;
- localAddress = "127.1.0.1";
- customResolver = {
- address = config.krebs.hosts.gum.nets.internet.ip4.addr;
- port = 15251;
- name = "2.dnscrypt-cert.euer.krebsco.de";
- key = "1AFC:E58D:F242:0FBB:9EE9:4E51:47F4:5373:D9AE:C2AB:DD96:8448:333D:5D79:272C:A44C";
- };
- };
- services.resolved.enable = true;
- services.resolved.fallbackDns = [ "127.1.0.1" ];
-}
diff --git a/lass/2configs/exim-smarthost.nix b/lass/2configs/exim-smarthost.nix
index 4455d2761..e05ed2427 100644
--- a/lass/2configs/exim-smarthost.nix
+++ b/lass/2configs/exim-smarthost.nix
@@ -79,6 +79,7 @@ with import <stockholm/lib>;
{ from = "ovh@lassul.us"; to = lass.mail; }
{ from = "hetzner@lassul.us"; to = lass.mail; }
{ from = "allygator@lassul.us"; to = lass.mail; }
+ { from = "immoscout@lassul.us"; to = lass.mail; }
];
system-aliases = [
{ from = "mailer-daemon"; to = "postmaster"; }
diff --git a/lass/2configs/gc.nix b/lass/2configs/gc.nix
index ad015180a..c5073e384 100644
--- a/lass/2configs/gc.nix
+++ b/lass/2configs/gc.nix
@@ -3,6 +3,6 @@
with import <stockholm/lib>;
{
nix.gc = {
- automatic = ! elem config.krebs.build.host.name [ "prism" "mors" "helios" ];
+ automatic = ! (elem config.krebs.build.host.name [ "prism" "mors" "helios" ] || config.boot.isContainer);
};
}
diff --git a/lass/2configs/git.nix b/lass/2configs/git.nix
index 1fe87c666..43085ba5e 100644
--- a/lass/2configs/git.nix
+++ b/lass/2configs/git.nix
@@ -57,6 +57,16 @@ let
cgit.desc = "Fork of nix-user-chroot my lethalman";
cgit.section = "software";
};
+ nixos-aws = {
+ collaborators = [ {
+ name = "fabio";
+ pubkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDFst8DvnfOu4pQJYxcwdf//jWTvP+jj0iSrOdt59c9Gbujm/8K1mBXhcSQhHj/GBRC1Qc1wipf9qZsWnEHMI+SRwq6tDr8gqlAcdWmHAs1bU96jJtc8EgmUKbXTFG/VmympMPi4cEbNUtH93v6NUjQKwq9szvDhhqSW4Y8zE32xLkySwobQapNaUrGAtQp3eTxu5Lkx+cEaaartaAspt8wSosXjUHUJktg0O5/XOP+CiWAx89AXxbQCy4XTQvUExoRGdw9sdu0lF0/A0dF4lFF/dDUS7+avY8MrKEcQ8Fwk8NcW1XrKMmCdNdpvou0whL9aHCdTJ+522dsSB1zZWh63Si4CrLKlc1TiGKCXdvzmCYrD+6WxbPJdRpMM4dFNtpAwhCm/dM+CBXfDkP0s5veFiYvp1ri+3hUqV/sep9r5/+d+5/R1gQs8WDNjWqcshveFbD5LxE6APEySB4QByGxIrw7gFbozE+PNxtlVP7bq4MyE6yIzL6ofQgO1e4THquPcqSCfCvyib5M2Q1phi5DETlMemWp84AsNkqbhRa4BGRycuOXXrBzE+RgQokcIY7t3xcu3q0xJo2+HxW/Lqi72zYU1NdT4nJMETEaG49FfIAnUuoVaQWWvOz8mQuVEmmdw2Yzo2ikILYSUdHTp1VPOeo6aNPvESkPw1eM0xDRlQ== ada";
+ } ];
+ };
+ krops = {
+ cgit.desc = "krebs deployment";
+ cgit.section = "software";
+ };
} // mapAttrs make-public-repo-silent {
};
@@ -70,8 +80,8 @@ let
import <secrets/repos.nix> { inherit config lib pkgs; }
);
- make-public-repo = name: { cgit ? {}, ... }: {
- inherit cgit name;
+ make-public-repo = name: { cgit ? {}, collaborators ? [], ... }: {
+ inherit cgit collaborators name;
public = true;
hooks = {
post-receive = pkgs.git-hooks.irc-announce {
diff --git a/lass/2configs/go.nix b/lass/2configs/go.nix
new file mode 100644
index 000000000..ecf89b298
--- /dev/null
+++ b/lass/2configs/go.nix
@@ -0,0 +1,19 @@
+{ config, lib, pkgs, ... }:
+{
+ krebs.go = {
+ enable = true;
+ };
+ services.nginx = {
+ enable = true;
+ virtualHosts.go = {
+ locations."/".extraConfig = ''
+ proxy_set_header Host go.lassul.us;
+ proxy_pass http://localhost:1337;
+ '';
+ serverAliases = [
+ "go.lassul.us"
+ ];
+ };
+ };
+}
+
diff --git a/lass/2configs/mail.nix b/lass/2configs/mail.nix
index 81db59617..2bb51b50a 100644
--- a/lass/2configs/mail.nix
+++ b/lass/2configs/mail.nix
@@ -206,8 +206,11 @@ in {
msmtp
mutt
pkgs.much
- pkgs.notmuch
tag-new-mails
tag-old-mails
];
+
+ nixpkgs.config.packageOverrides = opkgs: {
+ notmuch = (opkgs.notmuch.overrideAttrs (o: { doCheck = false; }));
+ };
}
diff --git a/lass/2configs/monitoring/client.nix b/lass/2configs/monitoring/client.nix
deleted file mode 100644
index b8c245215..000000000
--- a/lass/2configs/monitoring/client.nix
+++ /dev/null
@@ -1,26 +0,0 @@
-{pkgs, config, ...}:
-with import <stockholm/lib>;
-{
- services.telegraf = {
- enable = true;
-
- extraConfig = {
- agent.interval = "1s";
- outputs = {
- influxdb = {
- urls = ["http://prism:8086"];
- database = "telegraf_db";
- user_agent = "telegraf";
- };
- };
- inputs = {
- cpu = {
- percpu = false;
- totalcpu = true;
- };
- mem = {};
- net = {};
- };
- };
- };
-}
diff --git a/lass/2configs/monitoring/monit-alarms.nix b/lass/2configs/monitoring/monit-alarms.nix
deleted file mode 100644
index 2cfc292e5..000000000
--- a/lass/2configs/monitoring/monit-alarms.nix
+++ /dev/null
@@ -1,44 +0,0 @@
-{pkgs, config, ...}:
-with import <stockholm/lib>;
-let
- echoToIrc = msg:
- pkgs.writeDash "echo_irc" ''
- set -euf
- export LOGNAME=prism-alarm
- ${pkgs.irc-announce}/bin/irc-announce \
- irc.r 6667 ${config.networking.hostName}-alarm \#noise "${msg}" >/dev/null
- '';
-
-in {
- krebs.monit = {
- enable = true;
- http.enable = true;
- alarms = {
- nirwanabluete = {
- test = "${pkgs.curl}/bin/curl -sf 'https://nirwanabluete.de/'";
- alarm = echoToIrc "test nirwanabluete failed";
- };
- ubik = {
- test = "${pkgs.curl}/bin/curl -sf 'https://ubikmedia.de'";
- alarm = echoToIrc "test ubik failed";
- };
- cac-panel = {
- test = "${pkgs.curl}/bin/curl -sf 'https://panel.cloudatcost.com/login.php'";
- alarm = echoToIrc "test cac-panel failed";
- };
- radio = {
- test = pkgs.writeBash "check_stream" ''
- ${pkgs.curl}/bin/curl -sif http://lassul.us:8000/radio.ogg \
- | ${pkgs.gawk}/bin/awk '/^\r$/{exit}{print $0}' \
- | ${pkgs.gnugrep}/bin/grep -q "200 OK" || exit "''${PIPESTATUS[0]}"
- '';
- alarm = echoToIrc "test radio failed";
- };
- };
- };
-
- krebs.iptables.tables.filter.INPUT.rules = [
- { predicate = "-p tcp -i retiolum --dport 9093"; target = "ACCEPT"; }
- ];
-}
-
diff --git a/lass/2configs/monitoring/node-exporter.nix b/lass/2configs/monitoring/node-exporter.nix
new file mode 100644
index 000000000..561e3a25c
--- /dev/null
+++ b/lass/2configs/monitoring/node-exporter.nix
@@ -0,0 +1,15 @@
+{ config, lib, pkgs, ... }:
+{
+ krebs.iptables.tables.filter.INPUT.rules = [
+ { predicate = "-i retiolum -p tcp --dport 9100 -s ${config.krebs.hosts.prism.nets.retiolum.ip4.addr}"; target = "ACCEPT"; v6 = false; }
+ { predicate = "-i retiolum -p tcp --dport 9100 -s ${config.krebs.hosts.prism.nets.retiolum.ip6.addr}"; target = "ACCEPT"; v4 = false; }
+ ];
+ services.prometheus.exporters = {
+ node = {
+ enable = true;
+ enabledCollectors = [
+ "systemd"
+ ];
+ };
+ };
+}
diff --git a/lass/2configs/monitoring/prometheus-server.nix b/lass/2configs/monitoring/prometheus-server.nix
new file mode 100644
index 000000000..e16d421a0
--- /dev/null
+++ b/lass/2configs/monitoring/prometheus-server.nix
@@ -0,0 +1,217 @@
+{ pkgs, lib, config, ... }:
+{
+ #networking = {
+ # firewall.allowedTCPPorts = [
+ # 3000 # grafana
+ # 9090 # prometheus
+ # 9093 # alertmanager
+ # ];
+ # useDHCP = true;
+ #};
+
+ krebs.iptables.tables.filter.INPUT.rules = [
+ { predicate = "-i retiolum -p tcp --dport 3000"; target = "ACCEPT"; }
+ { predicate = "-i retiolum -p tcp --dport 9090"; target = "ACCEPT"; }
+ { predicate = "-i retiolum -p tcp --dport 9093"; target = "ACCEPT"; }
+ ];
+
+ services = {
+ prometheus = {
+ enable = true;
+ extraFlags = [
+ "-storage.local.retention 8760h"
+ "-storage.local.series-file-shrink-ratio 0.3"
+ "-storage.local.memory-chunks 2097152"
+ "-storage.local.max-chunks-to-persist 1048576"
+ "-storage.local.index-cache-size.fingerprint-to-metric 2097152"
+ "-storage.local.index-cache-size.fingerprint-to-timerange 1048576"
+ "-storage.local.index-cache-size.label-name-to-label-values 2097152"
+ "-storage.local.index-cache-size.label-pair-to-fingerprints 41943040"
+ ];
+ alertmanagerURL = [ "http://localhost:9093" ];
+ rules = [
+ ''
+ ALERT node_down
+ IF up == 0
+ FOR 5m
+ LABELS {
+ severity="page"
+ }
+ ANNOTATIONS {
+ summary = "{{$labels.alias}}: Node is down.",
+ description = "{{$labels.alias}} has been down for more than 5 minutes."
+ }
+ ALERT node_systemd_service_failed
+ IF node_systemd_unit_state{state="failed"} == 1
+ FOR 4m
+ LABELS {
+ severity="page"
+ }
+ ANNOTATIONS {
+ summary = "{{$labels.alias}}: Service {{$labels.name}} failed to start.",
+ description = "{{$labels.alias}} failed to (re)start service {{$labels.name}}."
+ }
+ ALERT node_filesystem_full_90percent
+ IF sort(node_filesystem_free{device!="ramfs"} < node_filesystem_size{device!="ramfs"} * 0.1) / 1024^3
+ FOR 5m
+ LABELS {
+ severity="page"
+ }
+ ANNOTATIONS {
+ summary = "{{$labels.alias}}: Filesystem is running out of space soon.",
+ description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% space left on its filesystem."
+ }
+ ALERT node_filesystem_full_in_4h
+ IF predict_linear(node_filesystem_free{device!="ramfs"}[1h], 4*3600) <= 0
+ FOR 5m
+ LABELS {
+ severity="page"
+ }
+ ANNOTATIONS {
+ summary = "{{$labels.alias}}: Filesystem is running out of space in 4 hours.",
+ description = "{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} is running out of space of in approx. 4 hours"
+ }
+ ALERT node_filedescriptors_full_in_3h
+ IF predict_linear(node_filefd_allocated[1h], 3*3600) >= node_filefd_maximum
+ FOR 20m
+ LABELS {
+ severity="page"
+ }
+ ANNOTATIONS {
+ summary = "{{$labels.alias}} is running out of available file descriptors in 3 hours.",
+ description = "{{$labels.alias}} is running out of available file descriptors in approx. 3 hours"
+ }
+ ALERT node_load1_90percent
+ IF node_load1 / on(alias) count(node_cpu{mode="system"}) by (alias) >= 0.9
+ FOR 1h
+ LABELS {
+ severity="page"
+ }
+ ANNOTATIONS {
+ summary = "{{$labels.alias}}: Running on high load.",
+ description = "{{$labels.alias}} is running with > 90% total load for at least 1h."
+ }
+ ALERT node_cpu_util_90percent
+ IF 100 - (avg by (alias) (irate(node_cpu{mode="idle"}[5m])) * 100) >= 90
+ FOR 1h
+ LABELS {
+ severity="page"
+ }
+ ANNOTATIONS {
+ summary = "{{$labels.alias}}: High CPU utilization.",
+ description = "{{$labels.alias}} has total CPU utilization over 90% for at least 1h."
+ }
+ ALERT node_ram_using_90percent
+ IF node_memory_MemFree + node_memory_Buffers + node_memory_Cached < node_memory_MemTotal * 0.1
+ FOR 30m
+ LABELS {
+ severity="page"
+ }
+ ANNOTATIONS {
+ summary="{{$labels.alias}}: Using lots of RAM.",
+ description="{{$labels.alias}} is using at least 90% of its RAM for at least 30 minutes now.",
+ }
+ ALERT node_swap_using_80percent
+ IF node_memory_SwapTotal - (node_memory_SwapFree + node_memory_SwapCached) > node_memory_SwapTotal * 0.8
+ FOR 10m
+ LABELS {
+ severity="page"
+ }
+ ANNOTATIONS {
+ summary="{{$labels.alias}}: Running out of swap soon.",
+ description="{{$labels.alias}} is using 80% of its swap space for at least 10 minutes now."
+ }
+ ''
+ ];
+ scrapeConfigs = [
+ {
+ job_name = "node";
+ scrape_interval = "10s";
+ static_configs = [
+ {
+ targets = [
+ ] ++ map (host: "${host}:9100") (lib.attrNames (lib.filterAttrs (_: host: host.owner.name == "lass" && host.monitoring) config.krebs.hosts));
+ #labels = {
+ # alias = "prometheus.example.com";
+ #};
+ }
+ ];
+ }
+ ];
+ alertmanager = {
+ enable = true;
+ listenAddress = "0.0.0.0";
+ configuration = {
+ "global" = {
+ "smtp_smarthost" = "smtp.example.com:587";
+ "smtp_from" = "alertmanager@example.com";
+ };
+ "route" = {
+ "group_by" = [ "alertname" "alias" ];
+ "group_wait" = "30s";
+ "group_interval" = "2m";
+ "repeat_interval" = "4h";
+ "receiver" = "team-admins";
+ };
+ "receivers" = [
+ {
+ "name" = "team-admins";
+ "email_configs" = [
+ {
+ "to" = "devnull@example.com";
+ "send_resolved" = true;
+ }
+ ];
+ "webhook_configs" = [
+ {
+ "url" = "http://127.0.0.1:14813/prometheus-alerts";
+ "send_resolved" = true;
+ }
+ ];
+ }
+ ];
+ };
+ };
+ };
+ grafana = {
+ enable = true;
+ addr = "0.0.0.0";
+ domain = "grafana.example.com";
+ rootUrl = "https://grafana.example.com/";
+ security = import <secrets/grafana_security.nix>; # { AdminUser = ""; adminPassword = ""}
+ };
+ };
+ services.logstash = {
+ enable = true;
+ inputConfig = ''
+ http {
+ port => 14813
+ host => "127.0.0.1"
+ }
+ '';
+ filterConfig = ''
+ if ([alerts]) {
+ ruby {
+ code => '
+ lines = []
+ event["alerts"].each {|p|
+ lines << "#{p["labels"]["instance"]}#{p["annotations"]["summary"]} #{p["status"]}"
+ }
+ event["output"] = lines.join("\n")
+ '
+ }
+ }
+ '';
+ outputConfig = ''
+ file { path => "/tmp/logs.json" codec => "json_lines" }
+ irc {
+ channels => [ "#noise" ]
+ host => "irc.r"
+ nick => "alarm"
+ codec => "json_lines"
+ format => "%{output}"
+ }
+ '';
+ #plugins = [ ];
+ };
+}
diff --git a/lass/2configs/monitoring/server.nix b/lass/2configs/monitoring/server.nix
deleted file mode 100644
index adaecde2c..000000000
--- a/lass/2configs/monitoring/server.nix
+++ /dev/null
@@ -1,87 +0,0 @@
-{pkgs, config, ...}:
-with import <stockholm/lib>;
-{
- services.influxdb.enable = true;
-
- services.influxdb.extraConfig = {
- meta.hostname = config.krebs.build.host.name;
- # meta.logging-enabled = true;
- http.bind-address = ":8086";
- admin.bind-address = ":8083";
- http.log-enabled = false;
- monitoring = {
- enabled = false;
- # write-interval = "24h";
- };
- collectd = [{
- enabled = true;
- typesdb = "${pkgs.collectd}/share/collectd/types.db";
- database = "collectd_db";
- port = 25826;
- }];
- };
-
- krebs.kapacitor =
- let
- db = "telegraf_db";
- echoToIrc = pkgs.writeDash "echo_irc" ''
- set -euf
- data="$(${pkgs.jq}/bin/jq -r .message)"
- export LOGNAME=prism-alarm
- ${pkgs.irc-announce}/bin/irc-announce \
- irc.r 6667 prism-alarm \#noise "$data" >/dev/null
- '';
- in {
- enable = true;
- alarms = {
- cpu = {