Merge remote-tracking branch 'cd/master'

54 files changed, 1418 insertions, 1120 deletions

diff --git a/Makefile b/Makefile
index 384c872ab..12a60a90b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,32 @@
-ifndef system
-$(error unbound variable: system)
+stockholm ?= .
+
+ifndef nixos-config
+$(if $(system),,$(error unbound variable: system))
+nixos-config = ./$(LOGNAME)/1systems/$(system).nix
+endif
+
+# target = [target_user@]target_host[:target_port][/target_path]
+ifdef target
+_target_user != echo $(target) | sed -n 's/@.*//p'
+_target_path != echo $(target) | sed -n 's/^[^/]*//p'
+_target_port != echo $(target) | sed -En 's|^.*:([^/]*)(/.*)?$$|\1|p'
+_target_host != echo $(target) | sed -En 's/^(.*@)?([^:/]*).*/\2/p'
+ifneq ($(_target_host),)
+$(if $(target_host),$(error cannot define both, target_host and host in target))
+target_host ?= $(_target_host)
+endif
+ifneq ($(_target_user),)
+$(if $(target_user),$(error cannot define both, target_user and user in target))
+target_user ?= $(_target_user)
+endif
+ifneq ($(_target_port),)
+$(if $(target_port),$(error cannot define both, target_port and port in target))
+target_port ?= $(_target_port)
+endif
+ifneq ($(_target_path),)
+$(if $(target_path),$(error cannot define both, target_path and path in target))
+target_path ?= $(_target_path)
+endif
 endif
 
 export target_host ?= $(system)
@@ -7,29 +34,37 @@ export target_user ?= root
 export target_port ?= 22
 export target_path ?= /var/src
 
+$(if $(target_host),,$(error unbound variable: target_host))
+$(if $(target_user),,$(error unbound variable: target_user))
+$(if $(target_port),,$(error unbound variable: target_port))
+$(if $(target_path),,$(error unbound variable: target_path))
+
 evaluate = \
 	nix-instantiate \
 		--eval \
 		--readonly-mode \
 		--show-trace \
-		-I nixos-config=./$(LOGNAME)/1systems/$(system).nix \
-		-I stockholm=. \
-		$(1)
+		-I nixos-config=$(nixos-config) \
+		-I stockholm=$(stockholm) \
+		-E '{ eval, f }: f eval' \
+		--arg eval 'import ./.' \
+		--arg f "eval@{ config, ... }: $(1)"
 
 execute = \
-	result=$$($(call evaluate,-A config.krebs.build.$(1) --json)) && \
+	result=$$($(call evaluate,config.krebs.build.$(1))) && \
 	script=$$(echo "$$result" | jq -r .) && \
-	echo "$$script" | sh
+	echo "$$script" | PS5=% sh
 
 # usage: make deploy system=foo [target_host=bar]
+deploy: ssh ?= ssh
 deploy:
 	$(call execute,populate)
-	ssh $(target_user)@$(target_host) -p $(target_port) \
+	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		nixos-rebuild switch --show-trace -I $(target_path)
 
 # usage: make LOGNAME=shared system=wolf eval.config.krebs.build.host.name
-eval eval.:;@$(call evaluate)
-eval.%:;@$(call evaluate,-A $*)
+eval eval.:;@$(call evaluate,$${expr-eval})
+eval.%:;@$(call evaluate,$*)
 
 # usage: make install system=foo [target_host=bar]
 install: ssh ?= ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
@@ -41,3 +76,21 @@ install:
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		env NIXOS_CONFIG=$(target_path)/nixos-config \
 			nixos-install
+
+# usage: make test system=foo [target=bar] [method={eval,build}]
+method ?= eval
+ifeq ($(method),build)
+test: command = nix-build --no-out-link
+else
+ifeq ($(method),eval)
+test: command ?= nix-instantiate --eval --json --readonly-mode --strict
+else
+$(error bad method: $(method))
+endif
+endif
+test: ssh ?= ssh
+test:
+	$(call execute,populate)
+	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
+		$(command) --show-trace -I $(target_path) \
+			-A config.system.build.toplevel $(target_path)/stockholm
diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix
index 0aa86dec9..97082f56a 100644
--- a/krebs/3modules/backup.nix
+++ b/krebs/3modules/backup.nix
@@ -117,6 +117,14 @@ let
               "$dst_user@$dst_host" \
               -T "$with_dst_path_lock_script"
         }
+        rsh="ssh -F /dev/null -i $identity ''${dst_port:+-p $dst_port}"
+        local_rsync() {
+          rsync "$@"
+        }
+        remote_rsync=${shell.escape (concatStringsSep " && " [
+          "mkdir -m 0700 -p ${shell.escape plan.dst.path}/current"
+          "exec flock -n ${shell.escape plan.dst.path} rsync"
+        ])}
       '';
       pull = ''
         identity=${shell.escape plan.dst.host.ssh.privkey.path}
@@ -131,6 +139,12 @@ let
         dst_shell() {
           eval "$with_dst_path_lock_script"
         }
+        rsh="ssh -F /dev/null -i $identity ''${src_port:+-p $src_port}"
+        local_rsync() {
+          mkdir -m 0700 -p ${shell.escape plan.dst.path}/current
+          flock -n ${shell.escape plan.dst.path} rsync "$@"
+        }
+        remote_rsync=rsync
       '';
     }}
     # Note that this only works because we trust date +%s to produce output
@@ -140,13 +154,10 @@ let
     with_dst_path_lock_script="exec env start_date=$(date +%s) "${shell.escape
       "flock -n ${shell.escape plan.dst.path} /bin/sh"
     }
-    rsync >&2 \
+    local_rsync >&2 \
         -aAXF --delete \
-        -e "ssh -F /dev/null -i $identity ''${dst_port:+-p $dst_port}" \
-        --rsync-path ${shell.escape (concatStringsSep " && " [
-          "mkdir -m 0700 -p ${shell.escape plan.dst.path}/current"
-          "exec flock -n ${shell.escape plan.dst.path} rsync"
-        ])} \
+        --rsh="$rsh" \
+        --rsync-path="$remote_rsync" \
         --link-dest="$dst_path/current" \
         "$src/" \
         "$dst/.partial"
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index a1f446188..d4c6b08df 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -20,35 +20,19 @@ let
       type = types.user;
     };
 
-    options.krebs.build.source = let
-      raw = types.either types.str types.path;
-      url = types.submodule {
+    options.krebs.build.source = mkOption {
+      type = with types; attrsOf (either str (submodule {
         options = {
-          url = mkOption {
-            type = types.str;
-          };
-          rev = mkOption {
-            type = types.str;
-          };
-          dev = mkOption {
-            type = types.str;
-          };
+          url = str;
+          rev = str;
         };
-      };
-    in mkOption {
-      type = types.attrsOf (types.either types.str url);
-      apply = let f = mapAttrs (_: value: {
-        string = value;
-        path = toString value;
-        set = f value;
-      }.${typeOf value}); in f;
+      }));
       default = {};
     };
 
     options.krebs.build.populate = mkOption {
       type = types.str;
       default = let
-        source = config.krebs.build.source;
         target-user = maybeEnv "target_user" "root";
         target-host = maybeEnv "target_host" config.krebs.build.host.name;
         target-port = maybeEnv "target_port" "22";
@@ -57,13 +41,16 @@ let
           #! /bin/sh
           set -eu
 
+          ssh=''${ssh-ssh}
+
           verbose() {
-            printf '+%s\n' "$(printf ' %q' "$@")" >&2
+            printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
             "$@"
           }
 
-          echo ${shell.escape git-script} \
-            | ssh -p ${shell.escape target-port} \
+          { printf 'PS5=%q%q\n' @ "$PS5"
+            echo ${shell.escape git-script}
+          } | verbose $ssh -p ${shell.escape target-port} \
                   ${shell.escape "${target-user}@${target-host}"} -T
 
           unset tmpdir
@@ -75,61 +62,34 @@ let
           tmpdir=$(mktemp -dt stockholm.XXXXXXXX)
           chmod 0755 "$tmpdir"
 
-          ${concatStringsSep "\n"
-            (mapAttrsToList
-              (name: spec: let dst = removePrefix "symlink:" (get-url spec); in
-                "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}")
-              symlink-specs)}
+          ${concatStringsSep "\n" (mapAttrsToList (name: symlink: ''
+            verbose ln -s ${shell.escape symlink.target} \
+                          "$tmpdir"/${shell.escape name}
+          '') source-by-method.symlink)}
 
           verbose proot \
-              -b $tmpdir:${shell.escape target-path} \
-              ${concatStringsSep " \\\n    "
-                (mapAttrsToList
-                  (name: spec:
-                    "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}")
-                  file-specs)} \
+              -b "$tmpdir":${shell.escape target-path} \
+              ${concatStringsSep " \\\n    " (mapAttrsToList (name: file:
+                "-b ${shell.escape "${file.path}:${target-path}/${name}"}"
+              ) source-by-method.file)} \
               rsync \
                   -f ${shell.escape "P /*"} \
-                  ${concatMapStringsSep " \\\n        "
-                    (name: "-f ${shell.escape "R /${name}"}")
-                    (attrNames file-specs)} \
+                  ${concatMapStringsSep " \\\n        " (name:
+                    "-f ${shell.escape "R /${name}"}"
+                  ) (attrNames source-by-method.file)} \
                   --delete \
                   -vFrlptD \
-                  -e ${shell.escape "ssh -p ${target-port}"} \
+                  -e "$ssh -p ${shell.escape target-port}" \
                   ${shell.escape target-path}/ \
                   ${shell.escape "${target-user}@${target-host}:${target-path}"}
         '';
 
-        get-schema = uri:
-          if substring 0 1 uri == "/"
-            then "file"
-            else head (splitString ":" uri);
-
-        has-schema = schema: uri: get-schema uri == schema;
-
-        get-url = spec: {
-          string = spec;
-          path = toString spec;
-          set = get-url spec.url;
-        }.${typeOf spec};
-
-        git-specs =
-          filterAttrs (_: spec: has-schema "https" (get-url spec)) source //
-          filterAttrs (_: spec: has-schema "http" (get-url spec)) source //
-          filterAttrs (_: spec: has-schema "git" (get-url spec)) source;
-
-        file-specs =
-          filterAttrs (_: spec: has-schema "file" (get-url spec)) source;
-
-        symlink-specs =
-          filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source;
-
         git-script = ''
           #! /bin/sh
           set -efu
 
           verbose() {
-            printf '+%s\n' "$(printf ' %q' "$@")" >&2
+            printf '%s%s\n' "$PS5$(printf ' %q' "$@")" >&2
             "$@"
           }
 
@@ -156,26 +116,48 @@ let
             if ! test "$(git log --format=%H -1)" = "$hash"; then
               git fetch origin
               git checkout "$hash" -- "$dst_dir"
-              git checkout "$hash"
+              git checkout -f "$hash"
             fi
 
             git clean -dxf
           )}
 
-          ${concatStringsSep "\n"
-            (mapAttrsToList
-              (name: spec: toString (map shell.escape [
-                "verbose"
-                "fetch_git"
-                "${target-path}/${name}"
-                spec.url
-                spec.rev
-              ]))
-              git-specs)}
+          ${concatStringsSep "\n" (mapAttrsToList (name: git: ''
+            verbose fetch_git ${concatMapStringsSep " " shell.escape [
+              "${target-path}/${name}"
+              git.url
+              git.rev
+            ]}
+          '') source-by-method.git)}
         '';
       in out;
     };
 
   };
 
+  source-by-method = let
+    known-methods = ["git" "file" "symlink"];
+  in genAttrs known-methods (const {}) // recursiveUpdate source-by-scheme {
+    git = source-by-scheme.http or {} //
+          source-by-scheme.https or {};
+  };
+
+  source-by-scheme = foldl' (out: { k, v }: recursiveUpdate out {
+    ${v.scheme}.${k} = v;
+  }) {} (mapAttrsToList (k: v: { inherit k v; }) normalized-source);
+
+  normalized-source = mapAttrs (name: let f = x: getAttr (typeOf x) {
+    path = f (toString x);
+    string = f {
+      url = if substring 0 1 x == "/" then "file://${x}" else x;
+    };
+    set = let scheme = head (splitString ":" x.url); in recursiveUpdate x {
+      inherit scheme;
+    } // {
+      symlink.target = removePrefix "symlink:" x.url;
+      file.path = # TODO file://host/...
+                  assert hasPrefix "file:///" x.url;
+                  removePrefix "file://" x.url;
+    }.${scheme} or {};
+  }; in f) config.krebs.build.source;
 in out
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index c06f3754e..186469e97 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -28,6 +28,7 @@ let
       ./realwallpaper.nix
       ./retiolum-bootstrap.nix
       ./retiolum.nix
+      ./secret.nix
       ./setuid.nix
       ./tinc_graphs.nix
       ./urlwatch.nix
@@ -42,9 +43,7 @@ let
 
     dns = {
       providers = mkOption {
-        # TODO with types; tree dns.label dns.provider, so we can merge.
-        # Currently providers can only be merged if aliases occur just once.
-        type = with types; attrsOf unspecified;
+        type = with types; attrsOf str;
       };
     };
 
@@ -94,7 +93,7 @@ let
     { krebs = import ./tv     { inherit config lib; }; }
     {
       krebs.dns.providers = {
-        de.krebsco = "zones";
+        "krebsco.de" = "zones";
         gg23 = "hosts";
         shack = "hosts";
         i = "hosts";
@@ -103,13 +102,27 @@ let
         retiolum = "hosts";
       };
 
-      networking.extraHosts = concatStringsSep "\n" (flatten (
+      krebs.users = {
+        krebs = {
+          home = "/krebs";
+          mail = "spam@krebsco.de";
+        };
+        root = {
+          home = "/root";
+          pubkey = config.krebs.build.host.ssh.pubkey;
+          uid = 0;
+        };
+      };
+
+      networking.extraHosts = let
+        domains = attrNames (filterAttrs (_: eq "hosts") cfg.dns.providers);
+        check = hostname: any (domain: hasSuffix ".${domain}" hostname) domains;
+      in concatStringsSep "\n" (flatten (
         mapAttrsToList (hostname: host:
           mapAttrsToList (netname: net:
             let
               aliases = longs ++ shorts;
-              providers = dns.split-by-provider net.aliases cfg.dns.providers;
-              longs = providers.hosts;
+              longs = filter check net.aliases;
               shorts = let s = ".${cfg.search-domain}"; in
                 map (removeSuffix s) (filter (hasSuffix s) longs);
             in
@@ -130,12 +143,11 @@ let
         { text=(stripEmptyLines value); }) all-zones;
 
       krebs.exim-smarthost.internet-aliases = let
-        format = from: to:
+        format = from: to: {
+          inherit from;
           # TODO assert is-retiolum-mail-address to;
-          { inherit from;
-            to = if typeOf to == "list"
-                   then concatMapStringsSep "," (getAttr "mail") to
-                   else to.mail; };
+          to = concatMapStringsSep "," (getAttr "mail") (toList to);
+        };
       in mapAttrsToList format (with config.krebs.users; let
         spam-ml = [
           lass
@@ -154,6 +166,10 @@ let
         "makefu@retiolum" = makefu;
         "spam@retiolum" = spam-ml;
         "tv@retiolum" = tv;
+        "lass@r" = lass;
+        "makefu@r" = makefu;
+        "spam@r" = spam-ml;
+        "tv@r" = tv;
       });
 
       services.openssh.hostKeys =
diff --git a/krebs/3modules/exim-retiolum.nix b/krebs/3modules/exim-retiolum.nix
index 1722eef1f..6e6928f89 100644
--- a/krebs/3modules/exim-retiolum.nix
+++ b/krebs/3modules/exim-retiolum.nix
@@ -11,6 +11,24 @@ let
 
   api = {
     enable = mkEnableOption "krebs.exim-retiolum";
+    local_domains = mkOption {
+      type = with types; listOf hostname;
+      default = ["localhost"] ++ config.krebs.build.host.nets.retiolum.aliases;
+    };
+    primary_hostname = mkOption {
+      type = types.str;
+      default = let x = "${config.krebs.build.host.name}.r"; in
+        assert elem x config.krebs.build.host.nets.retiolum.aliases;
+        x;
+    };
+    relay_to_domains = mkOption {
+      # TODO hostname with wildcards
+      type = with types; listOf str;
+      default = [
+        "*.r"
+        "*.retiolum"
+      ];
+    };
   };
 
   imp = {
@@ -21,9 +39,9 @@ let
         # TODO modular configuration
         assert config.krebs.retiolum.enable;
         ''
-          primary_hostname = ${retiolumHostname}
-          domainlist local_domains    = @ : localhost
-          domainlist relay_to_domains = *.retiolum
+          primary_hostname = ${cfg.primary_hostname}
+          domainlist local_domains = ${concatStringsSep ":" cfg.local_domains}
+          domainlist relay_to_domains = ${concatStringsSep ":" cfg.relay_to_domains}
           hostlist   relay_from_hosts = <; 127.0.0.1 ; ::1
 
           acl_smtp_rcpt = acl_check_rcpt
@@ -85,7 +103,7 @@ let
 
           retiolum:
             driver = manualroute
-            domains = ! ${retiolumHostname} : *.retiolum
+            domains = ! +local_domains : +relay_to_domains
             transport = remote_smtp
             route_list = ^.* $0 byname
             no_more
@@ -125,8 +143,8 @@ let
           # mode = 0660
 
           begin retry
-          *.retiolum             *           F,42d,1m
-          *                      *           F,2h,15m; G,16h,1h,1.5; F,4d,6h
+          ${concatMapStringsSep "\n" (k: "${k} * F,42d,1m") cfg.relay_to_domains}
+          * * F,2h,15m; G,16h,1h,1.5; F,4d,6h
 
           begin rewrite
 
@@ -134,8 +152,4 @@ let
         '';
     };
   };
-
-  # TODO get the hostname from somewhere else.
-  retiolumHostname = "${config.networking.hostName}.retiolum";
-in
-out
+in out
diff --git a/krebs/3modules/exim-smarthost.nix b/krebs/3modules/exim-smarthost.nix
index 267ee2900..c976e89de 100644
--- a/krebs/3modules/exim-smarthost.nix
+++ b/krebs/3modules/exim-smarthost.nix
@@ -25,14 +25,31 @@ let
       }));
     };
 
+    local_domains = mkOption {
+      type = with types; listOf hostname;
+      default = ["localhost"] ++ config.krebs.build.host.nets.retiolum.aliases;
+    };
+
     relay_from_hosts = mkOption {
       type = with types; listOf str;
       default = [];
+      apply = xs: ["127.0.0.1" "::1"] ++ xs;
+    };
+
+    relay_to_domains = mkOption {
+      # TODO hostname with wildcards
+      type = with types; listOf str;
+      default = [
+        "*.r"
+        "*.retiolum"
+      ];
     };
 
     primary_hostname = mkOption {
       type = types.str;
-      default = "${config.networking.hostName}.retiolum";
+      default = let x = "${config.krebs.build.host.name}.r"; in
+        assert elem x config.krebs.build.host.nets.retiolum.aliases;
+        x;
     };
 
     sender_domains = mkOption {
@@ -63,19 +80,11 @@ let
         # HOST_REDIR contains the real destinations for "local_domains".
         #HOST_REDIR = /etc/exim4/host_redirect
 
-
         # Domains not listed in local_domains need to be deliverable remotely.
         # XXX We abuse local_domains to mean "domains, we're the gateway for".
-        domainlist local_domains = @ : localhost
-        domainlist relay_to_domains =
-        hostlist relay_from_hosts = <;${concatStringsSep ";" (
-          [
-            "127.0.0.1"
-            "::1"
-          ]
-          ++
-          cfg.relay_from_hosts
-        )}
+        domainlist local_domains = ${concatStringsSep ":" cfg.local_domains}
+        domainlist relay_to_domains = ${concatStringsSep ":" cfg.relay_to_domains}
+        hostlist relay_from_hosts = <;${concatStringsSep ";" cfg.relay_from_hosts}
 
         acl_smtp_rcpt = acl_check_rcpt
         acl_smtp_data = acl_check_data
@@ -144,7 +153,7 @@ let
         retiolum:
           debug_print = "R: retiolum for $local_part@$domain"
           driver = manualroute
-          domains = ! ${cfg.primary_hostname} : *.retiolum
+          domains = ! +local_domains : +relay_to_domains
           transport = retiolum_smtp
           route_list = ^.* $0 byname
           no_more
@@ -197,8 +206,11 @@ let
           return_path_add
 
         begin retry
-        *.retiolum             *           F,42d,1m
-        *                      *           F,2h,15m; G,16h,1h,1.5; F,4d,6h
+        ${concatMapStringsSep "\n" (k: "${k} * F,42d,1m") cfg.relay_to_domains}
+        ${concatMapStringsSep "\n" (k: "${k} * F,42d,1m")
+                                   # TODO don't include relay_to_domains
+                                   (map (getAttr "from") cfg.internet-aliases)}
+        * * F,2h,15m; G,16h,1h,1.5; F,4d,6h
 
         begin rewrite
         begin authenticators
diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix
index a9542718d..0cc2f11c9 100644
--- a/krebs/3modules/git.nix
+++ b/krebs/3modules/git.nix
@@