Merge remote-tracking branch 'pnp/master'

7 files changed, 421 insertions, 9 deletions

diff --git a/krebs/3modules/bepasty-server.nix b/krebs/3modules/bepasty-server.nix
new file mode 100644
index 000000000..c99c3d11a
--- /dev/null
+++ b/krebs/3modules/bepasty-server.nix
@@ -0,0 +1,164 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  gunicorn = pkgs.pythonPackages.gunicorn;
+  bepasty = pkgs.pythonPackages.bepasty-server;
+  gevent = pkgs.pythonPackages.gevent;
+  python = pkgs.pythonPackages.python;
+  cfg = config.krebs.bepasty;
+
+  out = {
+    options.krebs.bepasty = api;
+    config = mkIf cfg.enable (mkMerge [
+      (mkIf cfg.serveNginx nginx-imp)
+      imp
+    ]);
+  };
+
+  api = {
+    enable = mkEnableOption "Bepasty Servers";
+    serveNginx = mkEnableOption "Serve Bepasty Servers with Nginx";
+
+    servers = mkOption {
+      type = with types; attrsOf optionSet;
+      options = singleton {
+        nginx = mkOption {
+          # TODO use the correct type
+          type = with types; attrsOf unspecified;
+          description = ''
+            additional nginx configuration. see krebs.nginx for all options
+          '';
+        };
+
+        secretKey = mkOption {
+          type = types.str;
+          description = ''
+            server secret for safe session cookies, must be set.
+          '';
+          default = "";
+        };
+
+        # we create a wsgi socket in $workDir/gunicorn-${name}.wsgi
+        workDir = mkOption {
+          type = types.str;
+          description = ''
+            Path to the working directory (used for sockets and pidfile).
+            Defaults to the users home directory. Must be accessible to nginx,
+            permissions will be set to 755
+          '';
+          default = config.users.extraUsers.bepasty.home;
+        };
+
+        dataDir = mkOption {
+          type = types.str;
+          description = ''
+            Defaults to the new users home dir which defaults to
+            /var/lib/bepasty-server/data
+          '';
+          default = "${config.users.extraUsers.bepasty.home}/data";
+        };
+
+        extraConfig = mkOption {
+          type = types.str;
+          default = "";
+          # TODO configure permissions in separate
+          example = ''
+            PERMISSIONS = {
+              'myadminsecret': 'admin,list,create,read,delete',
+            }
+            MAX_ALLOWED_FILE_SIZE = 5 * 1000 * 1000
+          '';
+        };
+
+        defaultPermissions = mkOption {
+          # TODO: listOf str
+          type = types.str;
+          description = ''
+            default permissions for all unauthenticated users.
+          '';
+          example = "read,create,delete";
+          default = "read";
+        };
+
+      };
+      default = {};
+    };
+
+  };
+
+  imp = {
+    # Configures systemd services for each configured server
+    # environment.systemPackages = [ bepasty gunicorn gevent ];
+    systemd.services = mapAttrs' (name: server:
+      nameValuePair "bepasty-server-${name}" {
+        description = "Bepasty Server ${name}";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        restartIfChanged = true;
+        environment = {
+          BEPASTY_CONFIG = "${server.workDir}/bepasty-${name}.conf";
+          PYTHONPATH= "${bepasty}/lib/${python.libPrefix}/site-packages:${gevent}/lib/${python.libPrefix}/site-packages";
+        };
+
+        serviceConfig = {
+          Type = "simple";
+          PrivateTmp = true;
+
+          ExecStartPre = assert server.secretKey != ""; pkgs.writeScript "bepasty-server.${name}-init" ''
+            #!/bin/sh
+            mkdir -p "${server.dataDir}" "${server.workDir}"
+            chown bepasty:bepasty "${server.workDir}" "${server.dataDir}"
+            cat > "${server.workDir}/bepasty-${name}.conf" <<EOF
+            SITENAME="${name}"
+            STORAGE_FILESYSTEM_DIRECTORY="${server.dataDir}"
+            SECRET_KEY="${server.secretKey}"
+            DEFAULT_PERMISSIONS="${server.defaultPermissions}"
+            ${server.extraConfig}
+            EOF
+          '';
+          ExecStart = ''${gunicorn}/bin/gunicorn bepasty.wsgi --name ${name} \
+            -u bepasty \
+            -g bepasty \
+            --workers 3 --log-level=info \
+            --bind=unix:${server.workDir}/gunicorn-${name}.sock \
+            --pid ${server.workDir}/gunicorn-${name}.pid \
+            -k gevent
+          '';
+        };
+      }
+    ) cfg.servers;
+
+    users.extraUsers.bepasty = {
+      uid = 2796546855; #genid bepasty
+      group = "bepasty";
+      home = "/var/lib/bepasty-server";
+    };
+    users.extraGroups.bepasty = {
+      gid = 2796546855; #genid bepasty
+    };
+  };
+
+  nginx-imp = {
+    assertions = [{ assertion = config.krebs.nginx.enable;
+                     message = "krebs.nginx.enable must be true"; }];
+
+    krebs.nginx.servers = mapAttrs' (name: server:
+      nameValuePair("bepasty-server-${name}")
+      (mkMerge [ server.nginx  {
+        extraConfig = ''
+          client_max_body_size 32M;
+          '';
+        locations = [
+          (nameValuePair "/" ''
+            proxy_set_header Host $http_host;
+            proxy_pass http://unix:${server.workDir}/gunicorn-${name}.sock;
+           '')
+          (nameValuePair "/static/" ''
+            alias ${bepasty}/lib/${python.libPrefix}/site-packages/bepasty/static/;
+          '')
+          ];
+      }])) cfg.servers ;
+  };
+in
+out
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index e2aea7057..e244ef7b7 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -6,6 +6,7 @@ let
 
   out = {
     imports = [
+      ./bepasty-server.nix
       ./build.nix
       ./exim-retiolum.nix
       ./exim-smarthost.nix
@@ -14,8 +15,10 @@ let
       ./iptables.nix
       ./nginx.nix
       ./Reaktor.nix
+      ./retiolum-bootstrap.nix
       ./realwallpaper.nix
       ./retiolum.nix
+      ./tinc_graphs.nix
       ./urlwatch.nix
     ];
     options.krebs = api;
@@ -102,10 +105,14 @@ let
 
       # Implements environment.etc."zones/<zone-name>"
       environment.etc = let
+        stripEmptyLines = s: concatStringsSep "\n"
+          (remove "\n" (remove "" (splitString "\n" s)));
         all-zones = foldAttrs (sum: current: sum + "\n" +current ) ""
-          ([cfg.zone-head-config] ++ combined-hosts) ;
+          ([cfg.zone-head-config] ++ combined-hosts);
         combined-hosts = (mapAttrsToList (name: value: value.extraZones)  cfg.hosts );
-      in lib.mapAttrs' (name: value: nameValuePair (("zones/" + name)) ({ text=value; })) all-zones;
+      in lib.mapAttrs' (name: value: nameValuePair
+        ("zones/" + name)
+        { text=(stripEmptyLines value); }) all-zones;
 
       krebs.exim-smarthost.internet-aliases = let
         format = from: to:
diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index 9cf5c9aea..4f8f964f8 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -127,10 +127,8 @@ with lib;
         "krebsco.de" = ''
                             IN MX 10  mx42
           euer              IN MX 1   aspmx.l.google.com.
-          io                IN NS     pigstarter.krebsco.de.
           pigstarter        IN A      ${head nets.internet.addrs4}
           gold              IN A      ${head nets.internet.addrs4}
-          tinc              IN A      ${head nets.internet.addrs4}
           boot              IN A      ${head nets.internet.addrs4}'';
       };
       nets = {
@@ -165,8 +163,11 @@ with lib;
       dc = "makefu"; #dc = "cac";
       extraZones = {
         "krebsco.de" = ''
-          wry            IN A ${head nets.internet.addrs4}
-          graphs         IN A ${head nets.internet.addrs4}
+          wry            IN A  ${head nets.internet.addrs4}
+          io             IN NS wry.krebsco.de.
+          graphs         IN A  ${head nets.internet.addrs4}
+          paste       60 IN A  ${head nets.internet.addrs4}
+          tinc           IN A  ${head nets.internet.addrs4}
           '';
       };
       nets = rec {
@@ -174,6 +175,7 @@ with lib;
           addrs4 = ["104.233.87.86"];
           aliases = [
             "wry.internet"
+            "paste.internet"
           ];
         };
         retiolum = {
@@ -182,6 +184,8 @@ with lib;
           addrs6 = ["42:6e1e:cc8a:7cef:827:f938:8c64:baad"];
           aliases = [
             "graphs.wry.retiolum"
+            "paste.wry.retiolum"
+            "paste.retiolum"
             "wry.retiolum"
           ];
           tinc.pubkey = ''
@@ -210,8 +214,7 @@ with lib;
         "krebsco.de" = ''
           omo               IN A      ${head nets.internet.addrs4}
           euer              IN A      ${head nets.internet.addrs4}
-          gum               IN A      ${head nets.internet.addrs4}
-          paste             IN A      ${head nets.internet.addrs4}'';
+          gum               IN A      ${head nets.internet.addrs4} '';
       };
       nets = {
         internet = {
diff --git a/krebs/3modules/nginx.nix b/krebs/3modules/nginx.nix
index 0530921a0..921771033 100644
--- a/krebs/3modules/nginx.nix
+++ b/krebs/3modules/nginx.nix
@@ -36,7 +36,7 @@ let
           type = with types; listOf (attrsOf str);
         };
         extraConfig = mkOption {
-          type = with types; str;
+          type = with types; string;
           default = "";
         };
       };
diff --git a/krebs/3modules/retiolum-bootstrap.nix b/krebs/3modules/retiolum-bootstrap.nix
new file mode 100644
index 000000000..65bb51193
--- /dev/null
+++ b/krebs/3modules/retiolum-bootstrap.nix
@@ -0,0 +1,64 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.krebs.retiolum-bootstrap;
+
+  out = {
+    options.krebs.retiolum-bootstrap = api;
+    config = mkIf cfg.enable imp ;
+  };
+
+  api = {
+    enable = mkEnableOption "retiolum boot strap for tinc.krebsco.de";
+    hostname = mkOption {
+        type = types.str;
+        description = "hostname which serves tinc boot";
+        default = "tinc.krebsco.de" ;
+    };
+    listen = mkOption {
+        type = with types; listOf str;
+        description = ''Addresses to listen on (nginx-syntax).
+        ssl will be configured, http will be redirected to ssl.
+        Make sure to have at least 1 ssl port configured.
+        '';
+        default = [ "80" "443 ssl" ] ;
+    };
+    ssl_certificate_key = mkOption {
+        type = types.str;
+        description = "Certificate key to use for ssl";
+        default = "/root/secrets/tinc.krebsco.de.key";
+    };
+    ssl_certificate = mkOption {
+        type = types.str;
+        description = "Certificate file to use for ssl";
+        default = "/root/secrets/tinc.krebsco.de.crt" ;
+    };
+    # in use:
+    #  <secrets/tinc.krebsco.de.crt>
+    #  <secrets/tinc.krebsco.de.key>
+  };
+
+  imp = {
+    krebs.nginx.servers = assert config.krebs.nginx.enable; {
+      retiolum-boot-ssl = {
+        server-names = singleton cfg.hostname;
+        listen = cfg.listen;
+        extraConfig = ''
+          ssl_certificate ${cfg.ssl_certificate};
+          ssl_certificate_key ${cfg.ssl_certificate_key};
+
+          if ($scheme = http){
+            return 301 https://$server_name$request_uri;
+          }
+
+          root ${pkgs.retiolum-bootstrap};
+          try_files $uri $uri/retiolum.sh;
+        '';
+        locations = [];
+      };
+    };
+  };
+
+in
+out
diff --git a/krebs/3modules/tinc_graphs.nix b/krebs/3modules/tinc_graphs.nix
new file mode 100644
index 000000000..a6c628353
--- /dev/null
+++ b/krebs/3modules/tinc_graphs.nix
@@ -0,0 +1,145 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.krebs.tinc_graphs;
+  internal_dir = "${cfg.workingDir}/internal";
+  external_dir = "${cfg.workingDir}/external";
+
+  out = {
+    options.krebs.tinc_graphs = api;
+    config = mkIf cfg.enable imp ;
+  };
+
+  api = {
+    enable = mkEnableOption "tinc graphs";
+
+    geodbPath = mkOption {
+      type = types.str;
+      description = "Path to geocitydb, defaults to geolite-legacy";
+      default = "${pkgs.geolite-legacy}/share/GeoIP/GeoIPCity.dat";
+    };
+
+    nginx = {
+      enable = mkEnableOption "enable tinc_graphs to be served with nginx";
+
+      anonymous = {
+        server-names = mkOption {
+          type = with types; listOf str;
+          description = "hostnames which serve anonymous graphs";
+          default = [ "graphs.${config.krebs.build.host.name}" ];
+        };
+
+        listen = mkOption {
+          # use the type of the nginx listen option
+          type = with types; listOf str;
+          description = "listen address for anonymous graphs";
+          default = [ "80" ];
+        };
+
+      };
+
+      complete = {
+        server-names = mkOption {
+          type = with types; listOf str;
+          description = "hostname which serves complete graphs";
+          default = [ "graphs.${config.krebs.build.host.name}" ];
+        };
+
+        listen = mkOption {
+          type = with types; listOf str;
+          description = "listen address for complete graphs";
+          default = [ "127.0.0.1:80" ];
+        };
+
+      };
+    };
+
+    workingDir = mkOption {
+      type = types.str;
+      description = ''
+        Path to working dir, will create interal and external/.
+        Defaults to the new users home dir which defaults to
+        /var/cache/tinc_graphs'';
+      default = config.users.extraUsers.tinc_graphs.home;
+    };
+
+    timerConfig = mkOption {
+      type = with types; attrsOf str;
+      default = {
+        OnCalendar = "*:0/15";
+      };
+    };
+  };
+
+  imp = {
+    environment.systemPackages = [ pkgs.tinc_graphs];
+    systemd.timers.tinc_graphs = {
+      description = "Build Tinc Graphs via via timer";
+      wantedBy = [ "timers.target"];
+      timerConfig = cfg.timerConfig;
+    };
+    systemd.services.tinc_graphs = {
+      description = "Build Tinc Graphs";
+      environment = {
+        EXTERNAL_FOLDER = external_dir;
+        INTERNAL_FOLDER = internal_dir;
+        GEODB = cfg.geodbPath;
+        TINC_HOSTPATH=config.krebs.retiolum.hosts;
+      };
+
+      restartIfChanged = true;
+
+      serviceConfig = {
+        Type = "simple";
+
+        ExecStartPre = pkgs.writeScript "tinc_graphs-init" ''
+          #!/bin/sh
+          if ! test -e "${cfg.workingDir}/internal/index.html"; then
+            cp -fr "$(${pkgs.tinc_graphs}/bin/tincstats-static-dir)/internal/" "${internal_dir}"
+          fi
+        '';
+
+        ExecStart = "${pkgs.tinc_graphs}/bin/all-the-graphs";
+
+        ExecStartPost = pkgs.writeScript "tinc_graphs-post" ''
+          #!/bin/sh
+          # TODO: this may break if workingDir is set to something stupid
+          # this is needed because homedir is created with 700
+          chmod 755  "${cfg.workingDir}"
+        '';
+        PrivateTmp = "yes";
+
+        User = "root"; # tinc cannot be queried as user,
+                       #  seems to be a tinc-pre issue
+      };
+    };
+
+    users.extraUsers.tinc_graphs = {
+      uid = 3925439960; #genid tinc_graphs
+      home = "/var/spool/tinc_graphs";
+      createHome = true;
+    };
+
+    krebs.nginx.servers = mkIf cfg.nginx.enable {
+      tinc_graphs_complete = mkMerge [ cfg.nginx.complete  {
+        locations = [
+          (nameValuePair "/" ''
+            autoindex on;
+            root ${internal_dir};
+          '')
+        ];
+      }] ;
+      tinc_graphs_anonymous = mkMerge [ cfg.nginx.anonymous {
+        locations = [
+          (nameValuePair "/" ''
+            autoindex on;
+            root ${external_dir};
+          '')
+        ];
+      }];
+    };
+  };
+
+in
+out
diff --git a/krebs/5pkgs/retiolum-bootstrap/default.nix b/krebs/5pkgs/retiolum-bootstrap/default.nix
new file mode 100644
index 000000000..331b1cb7f
--- /dev/null
+++ b/krebs/5pkgs/retiolum-bootstrap/default.nix
@@ -0,0 +1,29 @@
+{ stdenv,lib,fetchurl, ... }:
+with lib;
+stdenv.mkDerivation rec {
+  name = "retiolum-bootstrap";
+  version = "4.2.3";
+
+
+  src = fetchurl {
+    url = https://raw.githubusercontent.com/krebscode/painload/master/retiolum/scripts/tinc_setup/new_install.sh;
+    sha256 = "03kmil8q2xm3rdm2jxyah7vww84pw6w01d0c3siid9zpn2j7la9s";
+  };
+
+  phases = [
+    "installPhase"
+  ];
+
+  installPhase = ''
+    mkdir -p "$out"
+    cp -a ${src} $out/retiolum.sh
+  '';
+
+  meta = {
+    description = "Retiolum boostrap scripts";
+    url = https://github.com/krebscode/painload;
+    license = licenses.wtfpl;
+    platforms = platforms.unix;
+    maintainers = with maintainers; [ makefu ];
+  };
+}