Merge branch 'master' of prism:stockholm

59 files changed, 912 insertions, 281 deletions

diff --git a/Makefile b/Makefile
index ce6205c5b..cab53d52a 100644
--- a/Makefile
+++ b/Makefile
@@ -51,13 +51,23 @@ $(if $(target_user),,$(error unbound variable: target_user))
 $(if $(target_port),,$(error unbound variable: target_port))
 $(if $(target_path),,$(error unbound variable: target_path))
 
+whatsupnix = \
+	if type whatsupnix >/dev/null 2>&1; then \
+	  whatsupnix $(1); \
+	else \
+	  cat; \
+	fi
+
 build = \
 	nix-build \
+		-Q \
 		--no-out-link \
 		--show-trace \
 		-I nixos-config=$(nixos-config) \
 		-I stockholm=$(stockholm) \
-		-E "with import <stockholm>; $(1)"
+		-E "with import <stockholm>; $(1)" \
+		$(2) \
+	|& $(call whatsupnix)
 
 evaluate = \
 	nix-instantiate \
@@ -66,7 +76,8 @@ evaluate = \
 		--show-trace \
 		-I nixos-config=$(nixos-config) \
 		-I stockholm=$(stockholm) \
-		-E "let eval = import <stockholm>; in with eval; $(1)"
+		-E "let eval = import <stockholm>; in with eval; $(1)" \
+		$(2)
 
 ifeq ($(MAKECMDGOALS),)
 $(error No goals specified)
@@ -84,11 +95,7 @@ deploy:
 	$(ssh) $(target_user)@$(target_host) -p $(target_port) \
 		env STOCKHOLM_VERSION="$$STOCKHOLM_VERSION" \
 			nixos-rebuild -Q $(rebuild-command) --show-trace -I $(target_path) \
-		|& if type whatsupnix 2>/dev/null; then \
-		     whatsupnix $(target_user)@$(target_host):$(target_port); \
-		   else \
-		     cat; \
-		   fi
+	|& $(call whatsupnix,$(target_user)@$(target_host):$(target_port))
 
 # usage: make populate system=foo
 populate: populate-target = \
@@ -126,10 +133,10 @@ install:
 # usage: make test system=foo [target=bar] [method={eval,build}]
 method ?= eval
 ifeq ($(method),build)
-test: command = nix-build --no-out-link
+test: test = $(call build,$(1),$(2))
 else
 ifeq ($(method),eval)
-test: command ?= nix-instantiate --eval --json --readonly-mode --strict
+test: test ?= $(call evaluate,$(1),$(2)) --json --strict | jq -r .
 else
 $(error bad method: $(method))
 endif
@@ -141,6 +148,4 @@ else
 test: wrapper = $(ssh) $(target_user)@$(target_host) -p $(target_port)
 endif
 test: populate
-	$(wrapper) \
-		$(command) --show-trace -I $(target_path) \
-			-A config.system.build.toplevel $(target_path)/stockholm
+	$(wrapper) $(call test,config.system.build.toplevel,-I $(target_path))
diff --git a/doc/Commit_Messages_Guideline.md b/doc/Commit_Messages_Guideline.md
new file mode 100644
index 000000000..e704ee575
--- /dev/null
+++ b/doc/Commit_Messages_Guideline.md
@@ -0,0 +1,53 @@
+# Commit Messages Guideline
+
+Commits SHOULD have the following format:
+
+```
+<namespace?> <component>: <change>
+
+<rationale>
+
+(<reference-name>: <reference-id>)?
+```
+
+## `<namespace>`
+Defines where the change took place. This can be omitted if the
+namespace is `krebs`. Namespaces may be shortened to one to four characters (
+lassulus -> lass, makefu -> make, tv -> tv, shared -> sha)
+
+## `<component>`
+Name of the component which was touched. `component` is
+rather fuzzy and may mean different things, just choose what would fit best.
+
+Here are a numbers of samples for defining the component:
+
+* Change `gum` in `krebs/3modules/makefu/default.nix`: `gum.r: change ip`
+* Change `prepare.sh` in `krebs/4libs/infest`: `infest: prepare stockholm ISO`
+* Remove `concat` in `krebs/5pkgs`: `concat: RIP`, this commit may like some `<rationale>`
+* Update `types` in `krebs/3modules`: `lib/types: add managed bool to host type`
+* Change host `gum` in `makefu/1systems/gum`: `ma gum.r: add taskserver`
+* Change `tinc` module in `krebs/3modules`: `tinc module: add option enableLegacy`
+
+## `<rationale>`
+Describe some trivia why the commit was done:
+```
+whatsupnix: init
+
+Import from https://github.com/NixOS/nix/issues/443#issuecomment-296752535
+```
+
+## `<reference>`
+Defines external resouces related to the commit:
+```
+Closes: #123533
+CVE: CVE-2016-00001
+URL: https://example.com/CVE-2016-00001
+```
+
+## Remarks
+As a general rule of thumb you can check out: https://www.slideshare.net/TarinGamberini/commit-messages-goodpractices
+Of course the pattern not always fits perfectly (for example for refactoring),
+just apply some common sense and define a useful commit message,
+like `refactor krebs.setuid`.
+
+
diff --git a/doc/makefu/logbook/install_fileleech.md b/doc/makefu/logbook/install_fileleech.md
new file mode 100644
index 000000000..15f8c1bca
--- /dev/null
+++ b/doc/makefu/logbook/install_fileleech.md
@@ -0,0 +1,17 @@
+# install fileleech
+
+```
+builder$ python3 host.py --create-ssh-keys --create-passwords fileleech
+iso$ fdisk /dev/sda # 3 partitions, grub,boot,crypt
+iso$ cryptsetup luksFormat /dev/sda3 --cipher aes-xts-plain64 -s 512 -h sha512
+iso$ cryptsetup luksAddKey /dev/sda3 hddkey
+iso$ cryptsetup luksOpen --keyfile-size=4096  -d /dev/disk/by-id/usb-Intuix_DiskOnKey_09A07360336198F8-0:0 /dev/disk/by-id/ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN-part3 luksroot
+iso$ mkfs.ext4 -Lnixboot /dev/sda2
+iso$ mkfs.ext4 -Lroot /dev/mapper/luksroot
+iso$ echo 1 > /proc/sys/net/ipv6/conf/enp8s0f0/disable_ipv6
+iso$ mount /dev/mapper/luksroot /mnt
+iso$ mkdir /mnt/boot
+iso$ mount /dev/sda2 /mnt/boot
+iso$ mkdir -p /mnt/var/src
+iso$ touch /mnt/var/src/.populate
+```
diff --git a/doc/makefu/logbook/transfer_gum.md b/doc/makefu/logbook/transfer_gum.md
new file mode 100644
index 000000000..5f9c88256
--- /dev/null
+++ b/doc/makefu/logbook/transfer_gum.md
@@ -0,0 +1,16 @@
+# transfer gum to new hosts
+
+```
+builder$ vim krebs/3modules/makefu/default.nix
+## update ip
+builder$ vim makefu/1systems/gum.nix
+## update hardware config
+
+old-gum$ rsync --progress -lprtvzF  . <newip>:/mnt/
+
+new-gum$ touch /mnt/var/src/.populate
+new-gum$ gdisk /dev/sda r;g;w # gpt to mbr
+
+builder$ make -C ~/stockholm system=gum target=vcygfnhdxyxr47zu.onion install
+
+```
diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix
index 51f192703..976d378f9 100644
--- a/krebs/3modules/build.nix
+++ b/krebs/3modules/build.nix
@@ -24,4 +24,6 @@ with import <stockholm/lib>;
       type = types.user;
     };
   };
+
+  config.krebs.build.source.stockholm.file = mkDefault (toString <stockholm>);
 }
diff --git a/krebs/3modules/makefu/default.nix b/krebs/3modules/makefu/default.nix
index 4c0ce0fe3..c517ac1d8 100644
--- a/krebs/3modules/makefu/default.nix
+++ b/krebs/3modules/makefu/default.nix
@@ -308,7 +308,6 @@ with import <stockholm/lib>;
       extraZones = {
         "krebsco.de" = ''
           wry               IN A      ${nets.internet.ip4.addr}
-          io                IN NS     wry.krebsco.de.
           tinc              IN A      ${nets.internet.ip4.addr}
         '';
       };
@@ -470,6 +469,7 @@ with import <stockholm/lib>;
           wiki.euer         IN A      ${nets.internet.ip4.addr}
           graph             IN A      ${nets.internet.ip4.addr}
           ghook             IN A      ${nets.internet.ip4.addr}
+          io                IN NS     gum.krebsco.de.
         '';
       };
       nets = rec {
diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix
index c06e5ddb1..463fa26ba 100644
--- a/krebs/3modules/urlwatch.nix
+++ b/krebs/3modules/urlwatch.nix
@@ -60,6 +60,7 @@ let
       description = "URL to watch.";
       example = [
         https://nixos.org/channels/nixos-unstable/git-revision
+        { url = http://localhost ; filter = "grep:important.*stuff"; }
       ];
       apply = map (x: getAttr (typeOf x) {
         set = x;
@@ -79,7 +80,8 @@ let
   };
 
   urlsFile = pkgs.writeText "urls"
-    (concatMapStringsSep "\n---\n" toJSON cfg.urls);
+    (concatMapStringsSep "\n---\n"
+      (x: toJSON (filterAttrs (n: v: n != "_module") x)) cfg.urls);
 
   hooksFile = cfg.hooksFile;
 
@@ -142,17 +144,6 @@ let
         PrivateTmp = "true";
         SyslogIdentifier = "urlwatch";
         Type = "oneshot";
-        ExecStartPre =
-          pkgs.writeDash "urlwatch-prestart" ''
-            set -euf
-
-            dataDir=$HOME
-
-            if ! test -e "$dataDir"; then
-              mkdir -m 0700 -p "$dataDir"
-              chown ${user.name}: "$dataDir"
-            fi
-          '';
         ExecStart = pkgs.writeDash "urlwatch" ''
           set -euf
 
@@ -185,6 +176,8 @@ let
     };
     users.extraUsers = singleton {
       inherit (user) name uid;
+      home = cfg.dataDir;
+      createHome = true;
     };
   };
 
diff --git a/krebs/5pkgs/simple/ucspi-tcp/chmod.patch b/krebs/5pkgs/simple/ucspi-tcp/chmod.patch
new file mode 100644
index 000000000..dd6933208
--- /dev/null
+++ b/krebs/5pkgs/simple/ucspi-tcp/chmod.patch
@@ -0,0 +1,15 @@
+diff --git a/hier.c b/hier.c
+index 5663ada..1d73b84 100644
+--- a/hier.c
++++ b/hier.c
+@@ -2,8 +2,8 @@
+ 
+ void hier()
+ {
+-  h(auto_home,-1,-1,02755);
+-  d(auto_home,"bin",-1,-1,02755);
++  h(auto_home,-1,-1,0755);
++  d(auto_home,"bin",-1,-1,0755);
+ 
+   c(auto_home,"bin","tcpserver",-1,-1,0755);
+   c(auto_home,"bin","tcprules",-1,-1,0755);
diff --git a/krebs/5pkgs/simple/ucspi-tcp/default.nix b/krebs/5pkgs/simple/ucspi-tcp/default.nix
new file mode 100644
index 000000000..3b043be06
--- /dev/null
+++ b/krebs/5pkgs/simple/ucspi-tcp/default.nix
@@ -0,0 +1,86 @@
+{ stdenv, fetchurl }:
+
+stdenv.mkDerivation rec {
+  name = "ucspi-tcp-0.88";
+
+  src = fetchurl {
+    url = "http://cr.yp.to/ucspi-tcp/${name}.tar.gz";
+    sha256 = "171yl9kfm8w7l17dfxild99mbf877a9k5zg8yysgb1j8nz51a1ja";
+  };
+
+  # Plain upstream tarball doesn't build, get patches from Debian
+  patches = [
+    (fetchurl {
+      url = "http://ftp.de.debian.org/debian/pool/main/u/ucspi-tcp/ucspi-tcp_0.88-3.diff.gz";
+      sha256 = "0mzmhz8hjkrs0khmkzs5i0s1kgmgaqz07h493bd5jj5fm5njxln6";
+    })
+    ./chmod.patch
+  ];
+
+  # Apply Debian patches
+  postPatch = ''
+    for fname in debian/diff/*.diff; do
+        echo "Applying patch $fname"
+        patch < "$fname"
+    done
+  '';
+
+  # The build system is weird; 'make install' doesn't install anything, instead
+  # it builds an executable called ./install (from C code) which installs
+  # binaries to the directory given on line 1 in ./conf-home.
+  #
+  # Also, assume getgroups and setgroups work, instead of doing a build time
+  # test that breaks on NixOS (I think because nixbld users lack CAP_SETGID
+  # capability).
+  preBuild = ''
+    echo "$out" > conf-home
+
+    echo "main() { return 0; }" > chkshsgr.c
+  '';
+
+  installPhase = ''
+    mkdir -p "$out/bin"
+    mkdir -p "$out/share/man/man1"
+
+    # run the newly built installer
+    ./install
+
+    # Install Debian man pages (upstream has none)
+    cp debian/ucspi-tcp-man/*.1 "$out/share/man/man1"
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Command-line tools for building TCP client-server applications";
+    longDescription = ''
+      tcpserver waits for incoming connections and, for each connection, runs a
+      program of your choice. Your program receives environment variables
+      showing the local and remote host names, IP addresses, and port numbers.
+
+      tcpserver offers a concurrency limit to protect you from running out of
+      processes and memory. When you are handling 40 (by default) simultaneous
+      connections, tcpserver smoothly defers acceptance of new connections.
+
+      tcpserver also provides TCP access control features, similar to
+      tcp-wrappers/tcpd's hosts.allow but much faster. Its access control rules
+      are compiled into a hashed format with cdb, so it can easily deal with
+      thousands of different hosts.
+
+      This package includes a recordio tool that monitors all the input and
+      output of a server.
+
+      tcpclient makes a TCP connection and runs a program of your choice. It
+      sets up the same environment variables as tcpserver.
+
+      This package includes several sample clients built on top of tcpclient:
+      who@, date@, finger@, http@, tcpcat, and mconnect.
+
+      tcpserver and tcpclient conform to UCSPI, the UNIX Client-Server Program
+      Interface, using the TCP protocol. UCSPI tools are available for several
+      different networks.
+    '';
+    homepage = http://cr.yp.to/ucspi-tcp.html;
+    license = licenses.publicDomain;
+    platforms = platforms.linux;
+    maintainers = [ maintainers.bjornfor ];
+  };
+}
diff --git a/krebs/5pkgs/simple/whatsupnix/whatsupnix.bash b/krebs/5pkgs/simple/whatsupnix/whatsupnix.bash
index 042763048..2ad9aadc9 100644
--- a/krebs/5pkgs/simple/whatsupnix/whatsupnix.bash
+++ b/krebs/5pkgs/simple/whatsupnix/whatsupnix.bash
@@ -17,25 +17,22 @@
 #   2     Build error; at least one failed derivation could be found.
 #
 
-GAWK=${GAWK:-gawk}
-NIX_STORE=${NIX_STORE:-nix-store}
-
 failed_drvs=$(mktemp --tmpdir whatsupnix.XXXXXXXX)
 trap 'rm -f -- "$failed_drvs"' EXIT
 
 exec >&2
 
-$GAWK -v failed_drvs="$failed_drvs" '
+gawk -v failed_drvs="$failed_drvs" '
   match($0, /^builder for ‘(\/nix\/store\/[^’]+\.drv)’ failed/, m) {
     print m[1] >> failed_drvs
   }
-  { print $0 }
+  { print $0; fflush("/dev/stdout") }
 '
 
 case $# in
   0)
     print_log() {
-      NIX_PAGER= $NIX_STORE -l "$1"
+      NIX_PAGER= nix-store -l "$1"
     }
     ;;
   1)
diff --git a/lass/1systems/icarus.nix b/lass/1systems/icarus.nix
index b869a67a7..13c517e3b 100644
--- a/lass/1systems/icarus.nix
+++ b/lass/1systems/icarus.nix
@@ -6,9 +6,9 @@ with import <stockholm/lib>;
     ../.
     ../2configs/retiolum.nix
     ../2configs/hw/tp-x220.nix
-    ../2configs/baseX.nix
     ../2configs/git.nix
     ../2configs/exim-retiolum.nix
+    ../2configs/baseX.nix
     ../2configs/browsers.nix
     ../2configs/programs.nix
     ../2configs/fetchWallpaper.nix
@@ -22,9 +22,9 @@ with import <stockholm/lib>;
     loader.grub.enable = true;
     loader.grub.version = 2;
     loader.grub.device = "/dev/sda";
-    loader.grub.enableCryptodisk = true;
+    loader.grub.efiSupport = true;
 
-    initrd.luks.devices = [ { name = "luksroot"; device = "/dev/sda2"; } ];
+    initrd.luks.devices = [ { name = "luksroot"; device = "/dev/sda3"; } ];
     initrd.luks.cryptoModules = [ "aes" "sha512" "sha1" "xts" ];
     initrd.availableKernelModules = [ "xhci_hcd" "ehci_pci" "ahci" "usb_storage" ];
   };
@@ -34,11 +34,14 @@ with import <stockholm/lib>;
       fsType = "btrfs";
       options = ["defaults" "noatime" "ssd" "compress=lzo"];
     };
-    "/bku" = {
-      device = "/dev/mapper/pool-bku";
-      fsType = "btrfs";
-      options = ["defaults" "noatime" "ssd" "compress=lzo"];
+    "/boot" = {
+      device = "/dev/sda2";
     };
+    #"/bku" = {
+    #  device = "/dev/mapper/pool-bku";
+    #  fsType = "btrfs";
+    #  options = ["defaults" "noatime" "ssd" "compress=lzo"];
+    #};
     "/home" = {
       device = "/dev/mapper/pool-home";
       fsType = "btrfs";
diff --git a/lass/1systems/mors.nix b/lass/1systems/mors.nix
index b9ab54503..0bfd54515 100644
--- a/lass/1systems/mors.nix
+++ b/lass/1systems/mors.nix
@@ -96,49 +96,30 @@ with import <stockholm/lib>;
     loader.grub.enable = true;
     loader.grub.version = 2;
     loader.grub.device = "/dev/sda";
+    loader.grub.efiSupport = true;
 
-    initrd.luks.devices = [ { name = "luksroot"; device = "/dev/sda2"; } ];
+    initrd.luks.devices = [ { name = "luksroot"; device = "/dev/sda3"; } ];
     initrd.luks.cryptoModules = [ "aes" "sha512" "sha1" "xts" ];
     initrd.availableKernelModules = [ "xhci_hcd" "ehci_pci" "ahci" "usb_storage" ];
   };
   fileSystems = {
     "/" = {
-      device = "/dev/big/nix";
-      fsType = "ext4";
+      device = "/dev/mapper/pool-root";
+      fsType = "btrfs";
+      options = ["defaults" "noatime" "ssd" "compress=lzo"];
     };
-
     "/boot" = {
-      device = "/dev/sda1";
-    };
-
-    "/mnt/loot" = {
-      device = "/dev/big/loot";
-      fsType = "ext4";
+      device = "/dev/sda2";
     };
-
+    #"/bku" = {
+    #  device = "/dev/mapper/pool-bku";
+    #  fsType = "btrfs";
+    #  options = ["defaults" "noatime" "ssd" "compress=lzo"];
+    #};
     "/home" = {
-      device = "/dev/big/home";
-      fsType = "ext4";
-    };
-
-    "/home/lass" = {
-      device = "/dev/big/home-lass";
-      fsType = "ext4";
-    };
-
-    "/home/games/.local/share/Steam" = {
-      device = "/dev/big/steam";
-      fsType = "ext4";
-    };
-
-    "/home/virtual/virtual" = {
-      device = "/dev/big/virtual";
-      fsType = "ext4";
-    };
-
-    "/mnt/conf" = {
-      device = "/dev/big/conf";
-      fsType = "ext4";
+      device = "/dev/mapper/pool-home";
+      fsType = "btrfs";
+      options = ["defaults" "noatime" "ssd" "compress=lzo"];
     };
     "/tmp" = {
       device = "tmpfs";
diff --git a/lass/1systems/prism.nix b/lass/1systems/prism.nix
index af847333d..531dec9df 100644
--- a/lass/1systems/prism.nix
+++ b/lass/1systems/prism.nix
@@ -229,6 +229,10 @@ in {
         OnUnitInactiveSec = "2min";
         RandomizedDelaySec = "2min";
       };
+      krebs.repo-sync.repos.nixpkgs.timerConfig = {
+        OnBootSec = "90min";
+        OnUnitInactiveSec = "24h";
+      };
     }
     {
       lass.usershadow = {
diff --git a/lass/1systems/shodan.nix b/lass/1systems/shodan.nix
index dca616936..044e2ccf8 100644
--- a/lass/1systems/shodan.nix
+++ b/lass/1systems/shodan.nix
@@ -13,6 +13,7 @@ with import <stockholm/lib>;
     ../2configs/programs.nix
     ../2configs/fetchWallpaper.nix
     ../2configs/backups.nix
+    ../2configs/wine.nix
     #{
     #  users.extraUsers = {
     #    root = {
diff --git a/lass/2configs/buildbot-standalone.nix b/lass/2configs/buildbot-standalone.nix
index e765ddbb4..6c2a92c08 100644
--- a/lass/2configs/buildbot-standalone.nix
+++ b/lass/2configs/buildbot-standalone.nix
@@ -56,51 +56,6 @@ in {
           mode='full'
       )
 
-      # TODO: get nixpkgs/stockholm paths from krebs
-      env_lass = {
-        "LOGNAME": "lass",
-        "NIX_REMOTE": "daemon",
-        "dummy_secrets": "true",
-      }
-      env_makefu = {
-        "LOGNAME": "makefu",
-        "NIX_REMOTE": "daemon",
-        "dummy_secrets": "true",
-      }
-      env_nin = {
-        "LOGNAME": "nin",
-        "NIX_REMOTE": "daemon",
-        "dummy_secrets": "true",
-      }
-      env_shared = {
-        "LOGNAME": "shared",
-        "NIX_REMOTE": "daemon",
-        "dummy_secrets": "true",
-      }
-      env_tv = {
-        "LOGNAME": "tv",
-        "NIX_REMOTE": "daemon",
-        "dummy_secrets": "true",
-      }
-
-      # prepare nix-shell
-      # the dependencies which are used by the test script
-      deps = [
-        "gnumake",
-        "jq",
-        "nix",
-        "(import <stockholm>).pkgs.populate",
-        "openssh"
-      ]
-      # TODO: --pure , prepare ENV in nix-shell command:
-      #                   SSL_CERT_FILE,LOGNAME,NIX_REMOTE
-      nixshell = [
-        "nix-shell",
-        "-I", "/var/src",
-        "-I", "stockholm=.",
-        "-p"
-      ] + deps + [ "--run" ]
-
       # prepare addShell function
       def addShell(factory,**kwargs):
         factory.addStep(steps.ShellCommand(**kwargs))
@@ -110,30 +65,35 @@ in {
         f = util.BuildFactor