Merge remote-tracking branch 'cd/master'

4 files changed, 334 insertions, 11 deletions

diff --git a/krebs/3modules/bepasty-server.nix b/krebs/3modules/bepasty-server.nix
new file mode 100644
index 00000000..c99c3d11
--- /dev/null
+++ b/krebs/3modules/bepasty-server.nix
@@ -0,0 +1,164 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  gunicorn = pkgs.pythonPackages.gunicorn;
+  bepasty = pkgs.pythonPackages.bepasty-server;
+  gevent = pkgs.pythonPackages.gevent;
+  python = pkgs.pythonPackages.python;
+  cfg = config.krebs.bepasty;
+
+  out = {
+    options.krebs.bepasty = api;
+    config = mkIf cfg.enable (mkMerge [
+      (mkIf cfg.serveNginx nginx-imp)
+      imp
+    ]);
+  };
+
+  api = {
+    enable = mkEnableOption "Bepasty Servers";
+    serveNginx = mkEnableOption "Serve Bepasty Servers with Nginx";
+
+    servers = mkOption {
+      type = with types; attrsOf optionSet;
+      options = singleton {
+        nginx = mkOption {
+          # TODO use the correct type
+          type = with types; attrsOf unspecified;
+          description = ''
+            additional nginx configuration. see krebs.nginx for all options
+          '';
+        };
+
+        secretKey = mkOption {
+          type = types.str;
+          description = ''
+            server secret for safe session cookies, must be set.
+          '';
+          default = "";
+        };
+
+        # we create a wsgi socket in $workDir/gunicorn-${name}.wsgi
+        workDir = mkOption {
+          type = types.str;
+          description = ''
+            Path to the working directory (used for sockets and pidfile).
+            Defaults to the users home directory. Must be accessible to nginx,
+            permissions will be set to 755
+          '';
+          default = config.users.extraUsers.bepasty.home;
+        };
+
+        dataDir = mkOption {
+          type = types.str;
+          description = ''
+            Defaults to the new users home dir which defaults to
+            /var/lib/bepasty-server/data
+          '';
+          default = "${config.users.extraUsers.bepasty.home}/data";
+        };
+
+        extraConfig = mkOption {
+          type = types.str;
+          default = "";
+          # TODO configure permissions in separate
+          example = ''
+            PERMISSIONS = {
+              'myadminsecret': 'admin,list,create,read,delete',
+            }
+            MAX_ALLOWED_FILE_SIZE = 5 * 1000 * 1000
+          '';
+        };
+
+        defaultPermissions = mkOption {
+          # TODO: listOf str
+          type = types.str;
+          description = ''
+            default permissions for all unauthenticated users.
+          '';
+          example = "read,create,delete";
+          default = "read";
+        };
+
+      };
+      default = {};
+    };
+
+  };
+
+  imp = {
+    # Configures systemd services for each configured server
+    # environment.systemPackages = [ bepasty gunicorn gevent ];
+    systemd.services = mapAttrs' (name: server:
+      nameValuePair "bepasty-server-${name}" {
+        description = "Bepasty Server ${name}";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        restartIfChanged = true;
+        environment = {
+          BEPASTY_CONFIG = "${server.workDir}/bepasty-${name}.conf";
+          PYTHONPATH= "${bepasty}/lib/${python.libPrefix}/site-packages:${gevent}/lib/${python.libPrefix}/site-packages";
+        };
+
+        serviceConfig = {
+          Type = "simple";
+          PrivateTmp = true;
+
+          ExecStartPre = assert server.secretKey != ""; pkgs.writeScript "bepasty-server.${name}-init" ''
+            #!/bin/sh
+            mkdir -p "${server.dataDir}" "${server.workDir}"
+            chown bepasty:bepasty "${server.workDir}" "${server.dataDir}"
+            cat > "${server.workDir}/bepasty-${name}.conf" <<EOF
+            SITENAME="${name}"
+            STORAGE_FILESYSTEM_DIRECTORY="${server.dataDir}"
+            SECRET_KEY="${server.secretKey}"
+            DEFAULT_PERMISSIONS="${server.defaultPermissions}"
+            ${server.extraConfig}
+            EOF
+          '';
+          ExecStart = ''${gunicorn}/bin/gunicorn bepasty.wsgi --name ${name} \
+            -u bepasty \
+            -g bepasty \
+            --workers 3 --log-level=info \
+            --bind=unix:${server.workDir}/gunicorn-${name}.sock \
+            --pid ${server.workDir}/gunicorn-${name}.pid \
+            -k gevent
+          '';
+        };
+      }
+    ) cfg.servers;
+
+    users.extraUsers.bepasty = {
+      uid = 2796546855; #genid bepasty
+      group = "bepasty";
+      home = "/var/lib/bepasty-server";
+    };
+    users.extraGroups.bepasty = {
+      gid = 2796546855; #genid bepasty
+    };
+  };
+
+  nginx-imp = {
+    assertions = [{ assertion = config.krebs.nginx.enable;
+                     message = "krebs.nginx.enable must be true"; }];
+
+    krebs.nginx.servers = mapAttrs' (name: server:
+      nameValuePair("bepasty-server-${name}")
+      (mkMerge [ server.nginx  {
+        extraConfig = ''
+          client_max_body_size 32M;
+          '';
+        locations = [
+          (nameValuePair "/" ''
+            proxy_set_header Host $http_host;
+            proxy_pass http://unix:${server.workDir}/gunicorn-${name}.sock;
+           '')
+          (nameValuePair "/static/" ''
+            alias ${bepasty}/lib/${python.libPrefix}/site-packages/bepasty/static/;
+          '')
+          ];
+      }])) cfg.servers ;
+  };
+in
+out
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index c695589c..e244ef7b 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -6,6 +6,7 @@ let
 
   out = {
     imports = [
+      ./bepasty-server.nix
       ./build.nix
       ./exim-retiolum.nix
       ./exim-smarthost.nix
@@ -17,6 +18,7 @@ let
       ./retiolum-bootstrap.nix
       ./realwallpaper.nix
       ./retiolum.nix
+      ./tinc_graphs.nix
       ./urlwatch.nix
     ];
     options.krebs = api;
@@ -103,10 +105,14 @@ let
 
       # Implements environment.etc."zones/<zone-name>"
       environment.etc = let
+        stripEmptyLines = s: concatStringsSep "\n"
+          (remove "\n" (remove "" (splitString "\n" s)));
         all-zones = foldAttrs (sum: current: sum + "\n" +current ) ""
-          ([cfg.zone-head-config] ++ combined-hosts) ;
+          ([cfg.zone-head-config] ++ combined-hosts);
         combined-hosts = (mapAttrsToList (name: value: value.extraZones)  cfg.hosts );
-      in lib.mapAttrs' (name: value: nameValuePair (("zones/" + name)) ({ text=value; })) all-zones;
+      in lib.mapAttrs' (name: value: nameValuePair
+        ("zones/" + name)
+        { text=(stripEmptyLines value); }) all-zones;
 
       krebs.exim-smarthost.internet-aliases = let
         format = from: to:
diff --git a/krebs/3modules/tinc_graphs.nix b/krebs/3modules/tinc_graphs.nix
new file mode 100644
index 00000000..a6c62835
--- /dev/null
+++ b/krebs/3modules/tinc_graphs.nix
@@ -0,0 +1,145 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.krebs.tinc_graphs;
+  internal_dir = "${cfg.workingDir}/internal";
+  external_dir = "${cfg.workingDir}/external";
+
+  out = {
+    options.krebs.tinc_graphs = api;
+    config = mkIf cfg.enable imp ;
+  };
+
+  api = {
+    enable = mkEnableOption "tinc graphs";
+
+    geodbPath = mkOption {
+      type = types.str;
+      description = "Path to geocitydb, defaults to geolite-legacy";
+      default = "${pkgs.geolite-legacy}/share/GeoIP/GeoIPCity.dat";
+    };
+
+    nginx = {
+      enable = mkEnableOption "enable tinc_graphs to be served with nginx";
+
+      anonymous = {
+        server-names = mkOption {
+          type = with types; listOf str;
+          description = "hostnames which serve anonymous graphs";
+          default = [ "graphs.${config.krebs.build.host.name}" ];
+        };
+
+        listen = mkOption {
+          # use the type of the nginx listen option
+          type = with types; listOf str;
+          description = "listen address for anonymous graphs";
+          default = [ "80" ];
+        };
+
+      };
+
+      complete = {
+        server-names = mkOption {
+          type = with types; listOf str;
+          description = "hostname which serves complete graphs";
+          default = [ "graphs.${config.krebs.build.host.name}" ];
+        };
+
+        listen = mkOption {
+          type = with types; listOf str;
+          description = "listen address for complete graphs";
+          default = [ "127.0.0.1:80" ];
+        };
+
+      };
+    };
+
+    workingDir = mkOption {
+      type = types.str;
+      description = ''
+        Path to working dir, will create interal and external/.
+        Defaults to the new users home dir which defaults to
+        /var/cache/tinc_graphs'';
+      default = config.users.extraUsers.tinc_graphs.home;
+    };
+
+    timerConfig = mkOption {
+      type = with types; attrsOf str;
+      default = {
+        OnCalendar = "*:0/15";
+      };
+    };
+  };
+
+  imp = {
+    environment.systemPackages = [ pkgs.tinc_graphs];
+    systemd.timers.tinc_graphs = {
+      description = "Build Tinc Graphs via via timer";
+      wantedBy = [ "timers.target"];
+      timerConfig = cfg.timerConfig;
+    };
+    systemd.services.tinc_graphs = {
+      description = "Build Tinc Graphs";
+      environment = {
+        EXTERNAL_FOLDER = external_dir;
+        INTERNAL_FOLDER = internal_dir;
+        GEODB = cfg.geodbPath;
+        TINC_HOSTPATH=config.krebs.retiolum.hosts;
+      };
+
+      restartIfChanged = true;
+
+      serviceConfig = {
+        Type = "simple";
+
+        ExecStartPre = pkgs.writeScript "tinc_graphs-init" ''
+          #!/bin/sh
+          if ! test -e "${cfg.workingDir}/internal/index.html"; then
+            cp -fr "$(${pkgs.tinc_graphs}/bin/tincstats-static-dir)/internal/" "${internal_dir}"
+          fi
+        '';
+
+        ExecStart = "${pkgs.tinc_graphs}/bin/all-the-graphs";
+
+        ExecStartPost = pkgs.writeScript "tinc_graphs-post" ''
+          #!/bin/sh
+          # TODO: this may break if workingDir is set to something stupid
+          # this is needed because homedir is created with 700
+          chmod 755  "${cfg.workingDir}"
+        '';
+        PrivateTmp = "yes";
+
+        User = "root"; # tinc cannot be queried as user,
+                       #  seems to be a tinc-pre issue
+      };
+    };
+
+    users.extraUsers.tinc_graphs = {
+      uid = 3925439960; #genid tinc_graphs
+      home = "/var/spool/tinc_graphs";
+      createHome = true;
+    };
+
+    krebs.nginx.servers = mkIf cfg.nginx.enable {
+      tinc_graphs_complete = mkMerge [ cfg.nginx.complete  {
+        locations = [
+          (nameValuePair "/" ''
+            autoindex on;
+            root ${internal_dir};
+          '')
+        ];
+      }] ;
+      tinc_graphs_anonymous = mkMerge [ cfg.nginx.anonymous {
+        locations = [
+          (nameValuePair "/" ''
+            autoindex on;
+            root ${external_dir};
+          '')
+        ];
+      }];
+    };
+  };
+
+in
+out
diff --git a/krebs/default.nix b/krebs/default.nix
index 0b055cd2..31a7f7d0 100644
--- a/krebs/default.nix
+++ b/krebs/default.nix
@@ -3,6 +3,7 @@
 , current-user-name
 , lib
 , stockholm
+, StrictHostKeyChecking ? "yes"
 }:
 
 let out = {
@@ -131,18 +132,22 @@ let out = {
           s:.*\(/nix/store/[a-z0-9]*-nix-[0-9.]\+/bin/nix-env\).*:\1:p;T;q
         ')
         echo "nix-env is $nix_env" >&2
-        getchrootpath() {(
+        findpkg() {(
           name=$1
-          path=$(find /mnt/nix/store \
+          path=$(find /nix/store \
               -mindepth 1 -maxdepth 1 -type d -name '*-'"$name"'-*' \
             | head -n 1 | sed s:^/mnt::)
-          echo "$name is $path" >&2
-          echo "$path"
+          if echo "$path" | grep .; then
+            echo "$name is $path" >&2
+          else
+            echo "Error: package not found: $name" >&2
+            exit 1
+          fi
         )}
-        cacert=$(getchrootpath cacert)
-        coreutils=$(getchrootpath coreutils)
-        env="$coreutils/bin/env \
-            SSL_CERT_FILE=$cacert/etc/ssl/certs/ca-bundle.crt"
+        cacert=$(findpkg cacert)
+        coreutils=$(findpkg coreutils)
+        cp "$cacert"/etc/ssl/certs/ca-bundle.crt /mnt/root/SSL_CERT_FILE
+        env="$coreutils/bin/env SSL_CERT_FILE=/root/SSL_CERT_FILE"
         sed -i '
           s:^NIX_PATH=:chroot $mountPoint '"$env"' &:
           s:^nix-env:'"$nix_env"':
@@ -260,7 +265,10 @@ let out = {
     in out;
 
   rootssh = target: script:
-    "ssh root@${target} -T ${doc ''
+    let
+      flags = "-o StrictHostKeyChecking=${StrictHostKeyChecking}";
+    in
+    "ssh ${flags} root@${target} -T ${doc ''
       set -efu
       ${script}
     ''}";