Merge 'cd/master' - update krebs.build.source

5 files changed, 351 insertions, 90 deletions

diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
index 01bb16a2b..a1f335905 100644
--- a/krebs/3modules/backup.nix
+++ b/krebs/3modules/backup.nix
@@ -28,9 +28,17 @@ let
             type = types.krebs.file-location;
           };
           startAt = mkOption {
-            type = types.str;
+            default = "hourly";
+            type = types.str; # TODO systemd.time(7)'s calendar event
           };
           snapshots = mkOption {
+            default = {
+              hourly   = { format = "%Y-%m-%dT%H"; retain =  4; };
+              daily    = { format = "%Y-%m-%d";    retain =  7; };
+              weekly   = { format = "%YW%W";       retain =  4; };
+              monthly  = { format = "%Y-%m";       retain = 12; };
+              yearly   = { format = "%Y";                       };
+            };
             type = types.attrsOf (types.submodule {
               options = {
                 format = mkOption {
@@ -284,3 +292,15 @@ let
   };
 
 in out
+# TODO ionice
+# TODO mail on failed push, pull
+# TODO mail on missing push
+# TODO don't cancel plans on activation
+#   also, don't hang while deploying at:
+#   starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer
+# TODO make sure /bku is properly mounted
+# TODO make sure that secure hosts cannot backup to insecure ones
+# TODO optionally only backup when src and dst are near enough :)
+# TODO try using btrfs for snapshots (configurable)
+# TODO warn if partial snapshots are found
+# TODO warn if unknown stuff is found in dst path
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 7f004cd81..00142acdd 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -28,48 +28,159 @@ let
       type = types.user;
     };
 
-    options.krebs.build.source.dir = mkOption {
-      type = let
-        default-host = config.krebs.current.host;
-      in types.attrsOf (types.submodule ({ config, ... }: {
+    options.krebs.build.source = let
+      raw = types.either types.str types.path;
+      url = types.submodule {
         options = {
-          host = mkOption {
-            type = types.host;
-            default = default-host;
-          };
-          path = mkOption {
-            type = types.str;
-          };
-          target-path = mkOption {
-            type = types.str;
-            default = "/root/${config._module.args.name}";
-          };
           url = mkOption {
             type = types.str;
-            default = "file://${config.host.name}${config.path}";
-          };
-        };
-      }));
-      default = {};
-    };
-
-    options.krebs.build.source.git = mkOption {
-      type = with types; attrsOf (submodule ({ config, ... }: {
-        options = {
-          url = mkOption {
-            type = types.str; # TODO must be shell safe
           };
           rev = mkOption {
             type = types.str;
           };
-          target-path = mkOption {
+          dev = mkOption {
             type = types.str;
-            default = "/root/${config._module.args.name}";
           };
         };
-      }));
+      };
+    in mkOption {
+      type = types.attrsOf (types.either types.str url);
+      apply = let f = mapAttrs (_: value: {
+        string = value;
+        path = toString value;
+        set = f value;
+      }.${typeOf value}); in f;
       default = {};
     };
+
+    options.krebs.build.populate = mkOption {
+      type = types.str;
+      default = let
+        source = config.krebs.build.source;
+        target-user = maybeEnv "target_user" "root";
+        target-host = maybeEnv "target_host" config.krebs.build.host.name;
+        target-path = maybeEnv "target_path" "/var/src";
+        out = ''
+          #! /bin/sh
+          set -eu
+
+          verbose() {
+            printf '+%s\n' "$(printf ' %q' "$@")" >&2
+            "$@"
+          }
+
+          echo ${shell.escape git-script} \
+            | ssh ${shell.escape "${target-user}@${target-host}"} -T
+
+          unset tmpdir
+          trap '
+            rm "$tmpdir"/*
+            rmdir "$tmpdir"
+            trap - EXIT INT QUIT
+          '        EXIT INT QUIT
+          tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
+          chmod 0755 "$tmpdir"
+
+          ${concatStringsSep "\n"
+            (mapAttrsToList
+              (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
+                "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
+              symlink-specs)}
+
+          verbose proot \
+              -b $tmpdir:${shell.escape target-path} \
+              ${concatStringsSep " \\\n    "
+                (mapAttrsToList
+                  (name: spec:
+                    "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
+                  file-specs)} \
+              rsync \
+                  -f ${shell.escape "P /*"} \
+                  ${concatMapStringsSep " \\\n        "
+                    (name: "-f ${shell.escape "R /${name}"}")
+                    (attrNames file-specs)} \
+                  --delete \
+                  -vFrlptD \
+                  ${shell.escape target-path}/ \
+                  ${shell.escape "${target-user}@${target-host}:${target-path}"}
+        '';
+
+        get-schema = uri:
+          if substring 0 1 uri == "/"
+            then "file"
+            else head (splitString ":" uri);
+
+        has-schema = schema: uri: get-schema uri == schema;
+
+        get-url = spec: {
+          string = spec;
+          path = toString spec;
+          set = get-url spec.url;
+        }.${typeOf spec};
+
+        git-specs =
+          filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
+          filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
+          filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
+
+        file-specs =
+          filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
+
+        symlink-specs =
+          filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
+
+        git-script = ''
+          #! /bin/sh
+          set -efu
+
+          verbose() {
+            printf '+%s\n' "$(printf ' %q' "$@")" >&2
+            "$@"
+          }
+
+          fetch_git() {(
+            dst_dir=$1
+            src_url=$2
+            src_ref=$3
+
+            if ! test -e "$dst_dir"; then
+              git clone "$src_url" "$dst_dir"
+            fi
+
+            cd "$dst_dir"
+
+            if ! url=$(git config remote.origin.url); then
+              git remote add origin "$src_url"
+            elif test "$url" != "$src_url"; then
+              git remote set-url origin "$src_url"
+            fi
+
+            # TODO resolve src_ref to commit hash
+            hash=$src_ref
+
+            if ! test "$(git log --format=%H -1)" = "$hash"; then
+              git fetch origin
+              git checkout "$hash" -- "$dst_dir"
+              git checkout "$hash"
+            fi
+
+            git clean -dxf
+          )}
+
+          ${concatStringsSep "\n"
+            (mapAttrsToList
+              (name: spec: toString (map shell.escape [
+                "verbose"
+                "fetch_git"
+                "${target-path}/${name}"
+                spec.url
+                spec.rev
+              ]))
+              git-specs)}
+        '';
+      in out;
+    };
+
   };
 
 in out
diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix
index e6267d7e6..7b28ffca8 100644
--- a/krebs/3modules/git.nix
+++ b/krebs/3modules/git.nix
@@ -27,7 +27,7 @@ let
       description = ''
           Enable cgit.
           Cgit is an attempt to create a fast web interface for the git version
-          control system, using a built in cache to decrease pressure on the 
+          control system, using a built in cache to decrease pressure on the
           git server.
           cgit in this module is being served via fastcgi nginx.This module
           deploys a http://cgit.<hostname> nginx configuration and enables nginx
@@ -44,48 +44,8 @@ let
       default = "/etc/git";
     };
     repos = mkOption {
-      type = types.attrsOf (types.submodule ({
-        options = {
-          desc = mkOption {
-            type = types.nullOr types.str;
-            default = null;
-            description = ''
-              Repository description.
-            '';
-          };
-          section = mkOption {
-            type = types.nullOr types.str;
-            default = null;
-            description = ''
-              Repository section.
-            '';
-          };
-          name = mkOption {
-            type = types.str;
-            description = ''
-              Repository name.
-            '';
-          };
-          hooks = mkOption {
-            type = types.attrsOf types.str;
-            default = {};
-            description = ''
-              Repository-specific hooks.
-            '';
-          };
-          public = mkOption {
-            type = types.bool;
-            default = false;
-            description = ''
-              Allow everybody to read the repository via HTTP if cgit enabled.
-            '';
-            # TODO allow every configured user to fetch the repository via SSH.
-          };
-        };
-      }));
-
+      type = types.attrsOf subtypes.repo;
       default = {};
-
       example = literalExample ''
         {
           testing = {
@@ -99,7 +59,6 @@ let
           testing2 = { name = "testing2"; };
         }
       '';
-
       description = ''
         Repositories.
       '';
@@ -121,30 +80,158 @@ let
       '';
     };
     rules = mkOption {
-      type = types.unspecified;
+      type = types.listOf subtypes.rule;
+      default = [];
+      example = literalExample ''
+        singleton {
+          user = [ config.krebs.users.tv ];
+          repo = [ testing ]; # see literal example of repos
+          perm = push "refs/*" (with lib.git; [
+            non-fast-forward create delete merge
+          ]);
+        }
+      '';
+      description = ''
+        Rules.
+      '';
     };
   };
 
+  # TODO put into krebs/4lib/types.nix?
+  subtypes = {
+    repo = types.submodule ({
+      options = {
+        collaborators = mkOption {
+          type = types.listOf types.user;
+          default = [];
+          description = ''
+            List of users that should be able to fetch from this repo.
+
+            This option is currently not used by krebs.git but instead can be
+            used to create rules.  See e.g. <stockholm/tv/2configs/git.nix> for
+            an example.
+          '';
+        };
+        desc = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          description = ''
+            Repository description.
+          '';
+        };
+        section = mkOption {
+          type = types.nullOr types.str;
+          default = null;
+          description = ''
+            Repository section.
+          '';
+        };
+        name = mkOption {
+          type = types.str;
+          description = ''
+            Repository name.
+          '';
+        };
+        hooks = mkOption {
+          type = types.attrsOf types.str;
+          default = {};
+          description = ''
+            Repository-specific hooks.
+          '';
+        };
+        public = mkOption {
+          type = types.bool;
+          default = false;
+          description = ''
+            Allow everybody to read the repository via HTTP if cgit enabled.
+          '';
+          # TODO allow every configured user to fetch the repository via SSH.
+        };
+      };
+    });
+    rule = types.submodule ({ config, ... }: {
+      options = {
+        user = mkOption {
+          type = types.listOf types.user;
+          description = ''
+            List of users this rule should apply to.
+            Checked by authorize-command.
+          '';
+        };
+        repo = mkOption {
+          type = types.listOf subtypes.repo;
+          description = ''
+            List of repos this rule should apply to.
+            Checked by authorize-command.
+          '';
+        };
+        perm = mkOption {
+          type = types.submodule {
+            # TODO generate enum argument from krebs/4lib/git.nix
+            options = {
+              allow-commands = mkOption {
+                type = types.listOf (types.enum (with git; [
+                  git-receive-pack
+                  git-upload-pack
+                ]));
+                default = [];
+                description = ''
+                  List of commands the rule's users are allowed to execute.
+                  Checked by authorize-command.
+                '';
+              };
+              allow-receive-ref = mkOption {
+                type = types.nullOr types.str;
+                default = null;
+                description = ''
+                  Ref that can receive objects.
+                  Checked by authorize-push.
+                '';
+              };
+              allow-receive-modes = mkOption {
+                type = types.listOf (types.enum (with git; [
+                  fast-forward
+                  non-fast-forward
+                  create
+                  delete
+                  merge
+                ]));
+                default = [];
+                description = ''
+                  List of allowed receive modes.
+                  Checked by pre-receive hook.
+                '';
+              };
+            };
+          };
+          description = ''
+            Permissions granted.
+          '';
+        };
+      };
+    });
+  };
+
   git-imp = {
     system.activationScripts.git-init = "${init-script}";
-    
+
     # TODO maybe put all scripts here and then use PATH?
     environment.etc."${etc-base}".source =
       scriptFarm "git-ssh-authorizers" {
-        authorize-command = makeAuthorizeScript (map ({ repo, user, perm }: [
-          (map getName (ensureList user))
-          (map getName (ensureList repo))
-          (map getName perm.allow-commands)
+        authorize-command = makeAuthorizeScript (map (rule: [
+          (map getName (ensureList rule.user))
+          (map getName (ensureList rule.repo))
+          (map getName rule.perm.allow-commands)
         ]) cfg.rules);
-    
-        authorize-push = makeAuthorizeScript (map ({ repo, user, perm }: [
-          (map getName (ensureList user))
-          (map getName (ensureList repo))
-          (ensureList perm.allow-receive-ref)
-          (map getName perm.allow-receive-modes)
-        ]) (filter (x: hasAttr "allow-receive-ref" x.perm) cfg.rules));
+
+        authorize-push = makeAuthorizeScript (map (rule: [
+          (map getName (ensureList rule.user))
+          (map getName (ensureList rule.repo))
+          (ensureList rule.perm.allow-receive-ref)
+          (map getName rule.perm.allow-receive-modes)
+        ]) (filter (rule: rule.perm.allow-receive-ref != null) cfg.rules));
       };
-    
+
     users.extraUsers = singleton rec {
       description = "Git repository hosting user";
       name = "git";
diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix
index 6fd1c4224..31c1a375a 100644
--- a/krebs/3modules/tv/default.nix
+++ b/krebs/3modules/tv/default.nix
@@ -247,6 +247,7 @@ with lib;
         };
       };
       secure = true;
+      ssh.privkey.path = <secrets/ssh.id_ed25519>;
       ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcJvu8JDVzObLUtlAQg9qVugthKSfitwCljuJ5liyHa";
     };
     xu = {
diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix
index 31cbfcf6e..7a9fb55fd 100644
--- a/krebs/3modules/urlwatch.nix
+++ b/krebs/3modules/urlwatch.nix
@@ -54,6 +54,10 @@ let
       example = [
         https://nixos.org/channels/nixos-unstable/git-revision
       ];
+      apply = map (x: getAttr (typeOf x) {
+        set = x;
+        string.url = x;
+      });
     };
     verbose = mkOption {
       type = types.bool;
@@ -64,7 +68,40 @@ let
     };
   };
 
-  urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls);
+  urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls);
+
+  configFile = toFile "urlwatch.yaml" (toJSON {
+    display = {
+      error = true;
+      new = true;
+      unchanged = false;
+    };
+    report = {
+      email = {
+        enabled = false;
+        from = "";
+        html = false;
+        smtp = {
+          host = "localhost";
+          keyring = true;
+          port = 25;
+          starttls = true;
+        };
+        subject = "{count} changes: {jobs}";
+        to = "";
+      };
+      html.diff = "unified";
+      stdout = {
+        color = true;
+        enabled = true;
+      };
+      text = {
+        details = true;
+        footer = true;
+        line_length = 75;
+      };
+    };
+  });
 
   imp = {
     systemd.timers.urlwatch = {
@@ -109,10 +146,15 @@ let
           from=${escapeShellArg cfg.from}
           mailto=${escapeShellArg cfg.mailto}
           urlsFile=${escapeShellArg urlsFile}
+          configFile=${escapeShellArg configFile}
 
           cd /tmp
 
-          urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || :
+          urlwatch \
+              ${optionalString cfg.verbose "-v"} \
+              --urls="$urlsFile" \
+              --config="$configFile" \
+            > changes || :
 
           if test -s changes; then
             date=$(date -R)