Merge remote-tracking branch 'lass/master'

51 files changed, 1082 insertions, 498 deletions

diff --git a/jeschli/2configs/emacs.nix b/jeschli/2configs/emacs.nix
index f866bbca9..1ede96232 100644
--- a/jeschli/2configs/emacs.nix
+++ b/jeschli/2configs/emacs.nix
@@ -4,7 +4,7 @@ let
   pkgsWithOverlay = import <nixpkgs-unstable> {
     overlays = [
       (import (builtins.fetchTarball {
-        url = https://github.com/nix-community/emacs-overlay/archive/master.tar.gz;
+        url = https://github.com/nix-community/emacs-overlay/archive/403c14c23be188b58c0b1bc197b428041d8a0cea.tar.gz;
       }))
     ];
   };
diff --git a/krebs/2configs/buildbot-stockholm.nix b/krebs/2configs/buildbot-stockholm.nix
index ca6e0922a..43dd96919 100644
--- a/krebs/2configs/buildbot-stockholm.nix
+++ b/krebs/2configs/buildbot-stockholm.nix
@@ -26,14 +26,12 @@
         "http://cgit.hotdog.r/krops"
         "http://cgit.ni.r/krops"
         "http://cgit.prism.r/krops"
-        "https://git.ingolf-wagner.de/krebs/krops.git"
         "https://github.com/krebs/krops.git"
       ];
       nix_writers.urls = [
         "http://cgit.hotdog.r/nix-writers"
         "http://cgit.ni.r/nix-writers"
         "http://cgit.prism.r/nix-writers"
-        "https://git.ingolf-wagner.de/krebs/nix-writers.git"
       ];
       stockholm.urls = [
         "http://cgit.enklave.r/stockholm"
diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix
index 7695667fd..50db0b971 100644
--- a/krebs/3modules/ci.nix
+++ b/krebs/3modules/ci.nix
@@ -52,7 +52,7 @@ let
                   "${url}",
                   workdir='${name}-${elemAt(splitString "." url) 1}', branches=True,
                   project='${name}',
-                  pollinterval=10
+                  pollinterval=100
               )
           )
         '') repo.urls)
diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix
index 6f06f4510..aa06a883d 100644
--- a/krebs/3modules/default.nix
+++ b/krebs/3modules/default.nix
@@ -50,7 +50,6 @@ let
       ./secret.nix
       ./setuid.nix
       ./shadow.nix
-      ./syncthing.nix
       ./tinc.nix
       ./tinc_graphs.nix
       ./urlwatch.nix
diff --git a/krebs/3modules/external/default.nix b/krebs/3modules/external/default.nix
index 1d73fade2..b437456ec 100644
--- a/krebs/3modules/external/default.nix
+++ b/krebs/3modules/external/default.nix
@@ -373,6 +373,30 @@ in {
       syncthing.id = "22NLFY5-QMRM3BH-76QIBYI-OPMKVGM-DU4FNZI-3KN2POF-V4WIC6M-2SFFUAC";
       nets = {};
     };
+    catalonia = {
+     owner = config.krebs.users.xkey;
+     nets = {
+       retiolum = {
+         ip4.addr = "10.243.13.12";
+         aliases = [ "catalonia.r" ];
+         tinc.pubkey = ''
+           -----BEGIN RSA PUBLIC KEY-----
+           MIICCgKCAgEAug+nej8/spuRHdzcfBYAuzUVoiq4YufmJqXSshvgf4aqjeVEt91Y
+           gT6iBN8IKnMjYk3bAS7MxmgiyVE17MQlaQi0RSYY47M8I9TvCYtWX/FcXuP9e6CA
+           VcalDUNpy2qNB+yEE8gMa8vDA3smKk/iK47jTtpWoPtvejLK/SCi8RdlYjKlOErE
+           Yl9mCniGD1WEYgdrjf6Nl7av6uuGYNibivIMkB2JyGwGGmzvP+oBFi2Cwarw8K2e
+           FK2VGrAfkgiP5rTPACHseoeCsJtRLozgzYzmS5M9XhP5ZoPkbtR/pL5btCwoCTlZ
+           HotmLVg4DezbPjNOBB9gtJF4UuzQjSPNY6K1VvvLOhDwXdyln82LuNcm9l+cy9y3
+           mGeSvqOouBugDqie6OpkF0KrRwlGQVwzwtnDohGd/5f7TbiPf1QjC+JP/m4mxZl3
+           zE0BCOct9b4hUc/CFto71CPlytSbTsMhfJAn8JxttGvsWIAj+dQ0iuLXfLDflWt6
+           sImmnOo28YInvFx6pKoxTwcV1AVrPWn5TSePhZM50dmzs0exltOISFECDhpPabU3
+           ZymRCze8fH9Z3SHxfxTlTZV7IaW2kpyyBe1KsWpM46gLPk5icX+Xc6mdGwbdGBpf
+           vDZ+BoHCjq9FfQrAu1+E83yCYyu+3fWrLSgYyrqjg0gPcCcnb1g6hqECAwEAAQ==
+           -----END RSA PUBLIC KEY-----
+         '';
+        };
+      };
+    };
   };
   users = {
     ciko = {
@@ -421,6 +445,7 @@ in {
       mail = "xq@shackspace.de";
       pubkey = ssh-for "xq";
     };
+    xkey = {};
     miaoski = {
     };
     filly = {
diff --git a/krebs/3modules/external/mic92.nix b/krebs/3modules/external/mic92.nix
index a748b1454..23ab4f684 100644
--- a/krebs/3modules/external/mic92.nix
+++ b/krebs/3modules/external/mic92.nix
@@ -144,24 +144,24 @@ in {
         };
       };
     };
-    idontcare = {
+    herbert = {
       owner = config.krebs.users.Mic92;
       nets = rec {
         retiolum = {
           addrs = [
-            config.krebs.hosts.idontcare.nets.retiolum.ip4.addr
-            config.krebs.hosts.idontcare.nets.retiolum.ip6.addr
+            config.krebs.hosts.herbert.nets.retiolum.ip4.addr
+            config.krebs.hosts.herbert.nets.retiolum.ip6.addr
           ];
           ip4.addr = "10.243.29.177";
-          aliases = [ "idontcare.r" ];
+          aliases = [ "herbert.r" ];
           tinc.pubkey = ''
             -----BEGIN RSA PUBLIC KEY-----
-            MIIBCgKCAQEAxmmbQLVXcnCU9Vg9TCoJxfq/RyNfzaTj8XJsn4Kpo3CvQOwFzL6O
-            qZnbG55WjPjPumuFgtUdHA/G8mgtrTVaIRbVE9ck2l2wWFzMWxORzuvDbMh5xP8A
-            OW2Z2qjlH6O9GTBCzpYyHuyBWCjtiN4x9zEqxkIsBARKOylAoy3zQIiiQF0d72An
-            lqKFi9vYUU90zo9rP8BTzx2ZsEWb28xhHUlwf1+vgaOHI1jI99gnr12dVYl/i/Hb
-            O28gDUogfpP/5pWFAHJ+53ZscHo8/Y7imjiKgGXmOHywoXOsKQ67M6ROEU/0xPnw
-            jKmq2p7zTJk2mDhphjePi5idd5yKNX5Q3wIDAQAB
+            MIIBCgKCAQEA7ZINr8YxVwHtcOR+ySpc9UjnJWsFXlOyu3CnrJ8IrY+mPA25UmNZ
+            stXd8QbJuxpad9HyPs294uW8UmXttEZzIwAlikVHasM5IQHVltudTTFvv7s3YFWd
+            /lgpHbo8zOA2mafx+Sr02Fy/lHjk6BTf8IOzdJIpUHZL/P+FUl9baBwGLmtbEvPh
+            fbvtf5QryBjJ9nRnb+wsPVpeFE/LncIMK/bYQsyE01T5QDu/muAaeYPbgm6FqaQH
+            OJ4oEHsarWBvU1qzgz/IRz0BHHeTrbbP3AG/glTwL02Z1mtTXSjME7cfk7ZRM5Cj
+            jXAqnqu2m1B08Kii+zYp4BPZDmPLT5gq+QIDAQAB
             -----END RSA PUBLIC KEY-----
           '';
         };
diff --git a/krebs/3modules/hidden-ssh.nix b/krebs/3modules/hidden-ssh.nix
index 2d697e497..f497de451 100644
--- a/krebs/3modules/hidden-ssh.nix
+++ b/krebs/3modules/hidden-ssh.nix
@@ -19,6 +19,10 @@ let
       type = types.str;
       default = "irc.freenode.org";
     };
+    message = mkOption {
+      type = types.str;
+      default = "SSH Hidden Service at ";
+    };
   };
 
   imp = let
@@ -50,7 +54,7 @@ let
             ${pkgs.irc-announce}/bin/irc-announce \
             ${cfg.server} 6667 ${config.krebs.build.host.name}-ssh \
             \${cfg.channel} \
-            "SSH Hidden Service at $(cat ${hiddenServiceDir}/hostname)"
+            "${cfg.message}$(cat ${hiddenServiceDir}/hostname)"
         '';
         PrivateTmp = "true";
         User = "tor";
diff --git a/krebs/3modules/reaktor2.nix b/krebs/3modules/reaktor2.nix
index 837a9bea7..fcc453fa4 100644
--- a/krebs/3modules/reaktor2.nix
+++ b/krebs/3modules/reaktor2.nix
@@ -45,6 +45,10 @@ with import <stockholm/lib>;
           default = self.config.port == "6697";
           type = types.bool;
         };
+        API.listen = mkOption {
+          default = null;
+          type = types.nullOr types.str;
+        };
       };
     }));
   };
@@ -65,9 +69,11 @@ with import <stockholm/lib>;
           ExecStart = let
             configFile = pkgs.writeJSON configFileName configValue;
             configFileName = "${cfg.systemd-service-name}.config.json";
-            configValue = recursiveUpdate {
-              logTime = false;
-            } (removeAttrs cfg ["_module"]);
+            configValue = stripAttr (
+              recursiveUpdate {
+                logTime = false;
+              } (removeAttrs cfg ["_module"])
+            );
           in "${pkgs.reaktor2}/bin/reaktor ${configFile}";
           Restart = "always";
           RestartSec = "30";
diff --git a/krebs/3modules/realwallpaper.nix b/krebs/3modules/realwallpaper.nix
index a83758ccd..cfa8a65ba 100644
--- a/krebs/3modules/realwallpaper.nix
+++ b/krebs/3modules/realwallpaper.nix
@@ -17,21 +17,6 @@ let
       default = "/var/realwallpaper/";
     };
 
-    nightmap = mkOption {
-      type = types.str;
-      default = "http://eoimages.gsfc.nasa.gov/images/imagerecords/55000/55167/earth_lights_lrg.jpg";
-    };
-
-    daymap = mkOption {
-      type = types.str;
-      default = "https://www.nnvl.noaa.gov/images/globaldata/SnowIceCover_Daily.png";
-    };
-
-    cloudmap = mkOption {
-      type = types.str;
-      default = "http://home.megapass.co.kr/~holywatr/cloud_data/clouds_2048.jpg";
-    };
-
     marker = mkOption {
       type = types.str;
       default = "http://graph.r/marker.json";
@@ -60,6 +45,7 @@ let
       path = with pkgs; [
         xplanet
         imagemagick
+        inkscape
         curl
         file
         jq
@@ -67,9 +53,6 @@ let
 
       environment = {
         working_dir = cfg.workingDir;
-        nightmap_url = cfg.nightmap;
-        daymap_url = cfg.daymap;
-        cloudmap_url = cfg.cloudmap;
         marker_url = cfg.marker;
       };
 
diff --git a/krebs/3modules/syncthing.nix b/krebs/3modules/syncthing.nix
deleted file mode 100644
index 799ed7eda..000000000
--- a/krebs/3modules/syncthing.nix
+++ /dev/null
@@ -1,206 +0,0 @@
-{ config, pkgs, ... }: with import <stockholm/lib>;
-
-let
-
-  kcfg = config.krebs.syncthing;
-  scfg = config.services.syncthing;
-
-  devices = mapAttrsToList (name: peer: {
-    name = name;
-    deviceID = peer.id;
-    addresses = peer.addresses;
-  }) kcfg.peers;
-
-  folders = mapAttrsToList ( _: folder: {
-    inherit (folder) path id type;
-    devices = map (peer: { deviceId = kcfg.peers.${peer}.id; }) folder.peers;
-    rescanIntervalS = folder.rescanInterval;
-    fsWatcherEnabled = folder.watch;
-    fsWatcherDelayS = folder.watchDelay;
-    ignoreDelete = folder.ignoreDelete;
-    ignorePerms = folder.ignorePerms;
-  }) kcfg.folders;
-
-  getApiKey = pkgs.writeDash "getAPIKey" ''
-    ${pkgs.libxml2}/bin/xmllint \
-      --xpath 'string(configuration/gui/apikey)'\
-      ${scfg.configDir}/config.xml
-  '';
-
-  updateConfig = pkgs.writeDash "merge-syncthing-config" ''
-    set -efu
-
-    # XXX this assumes the GUI address to be "IPv4 address and port"
-    host=${shell.escape (elemAt (splitString ":" scfg.guiAddress) 0)}
-    port=${shell.escape (elemAt (splitString ":" scfg.guiAddress) 1)}
-
-    # wait for service to restart
-    ${pkgs.untilport}/bin/untilport "$host" "$port"
-
-    API_KEY=$(${getApiKey})
-
-    _curl() {
-      ${pkgs.curl}/bin/curl \
-          -Ss \
-          -H "X-API-Key: $API_KEY" \
-          "http://$host:$port/rest""$@"
-    }
-
-    old_config=$(_curl /system/config)
-    new_config=${shell.escape (toJSON {
-      inherit devices folders;
-    })}
-    new_config=$(${pkgs.jq}/bin/jq -en \
-        --argjson old_config "$old_config" \
-        --argjson new_config "$new_config" \
-        '
-          $old_config * $new_config
-          ${optionalString (!kcfg.overridePeers) ''
-            * { devices: $old_config.devices }
-          ''}
-          ${optionalString (!kcfg.overrideFolders) ''
-            * { folders: $old_config.folders }
-          ''}
-        '
-    )
-    echo $new_config | _curl /system/config -d @-
-    _curl /system/restart -X POST
-  '';
-
-in
-
-{
-  options.krebs.syncthing = {
-
-    enable = mkEnableOption "syncthing-init";
-
-    cert = mkOption {
-      type = types.nullOr types.absolute-pathname;
-      default = null;
-    };
-
-    key = mkOption {
-      type = types.nullOr types.absolute-pathname;
-      default = null;
-    };
-
-    overridePeers = mkOption {
-      type = types.bool;
-      default = true;
-      description = ''
-        Whether to delete the peers which are not configured via the peers option
-      '';
-    };
-    peers = mkOption {
-      default = {};
-      type = types.attrsOf (types.submodule ({
-        options = {
-
-          # TODO make into addr + port submodule
-          addresses = mkOption {
-            type = types.listOf types.str;
-            default = [];
-          };
-
-          #TODO check
-          id = mkOption {
-            type = types.str;
-          };
-
-        };
-      }));
-    };
-
-    overrideFolders = mkOption {
-      type = types.bool;
-      default = true;
-      description = ''
-        Whether to delete the folders which are not configured via the peers option
-      '';
-    };
-    folders = mkOption {
-      default = {};
-      type = types.attrsOf (types.submodule ({ config, ... }: {
-        options = {
-
-          path = mkOption {
-            type = types.absolute-pathname;
-            default = config._module.args.name;
-          };
-
-          id = mkOption {
-            type = types.str;
-            default = config._module.args.name;
-          };
-
-          peers = mkOption {
-            type = types.listOf types.str;
-            default = [];
-          };
-
-          rescanInterval = mkOption {
-            type = types.int;
-            default = 3600;
-          };
-
-          type = mkOption {
-            type = types.enum [ "sendreceive" "sendonly" "receiveonly" ];
-            default = "sendreceive";
-          };
-
-          watch = mkOption {
-            type = types.bool;
-            default = true;
-          };
-
-          watchDelay = mkOption {
-            type = types.int;
-            default = 10;
-          };
-
-          ignoreDelete = mkOption {
-            type = types.bool;
-            default = false;
-          };
-
-          ignorePerms = mkOption {
-            type = types.bool;
-            default = true;
-          };
-
-        };
-      }));
-    };
-  };
-
-  config = mkIf kcfg.enable {
-
-    systemd.services.syncthing = mkIf (kcfg.cert != null || kcfg.key != null) {
-      serviceConfig.PermissionsStartOnly = mkDefault true;
-      preStart = ''
-        ${optionalString (kcfg.cert != null) ''
-          cp ${toString kcfg.cert} ${scfg.configDir}/cert.pem
-          chown ${scfg.user}:${scfg.group} ${scfg.configDir}/cert.pem
-          chmod 400 ${scfg.configDir}/cert.pem
-        ''}
-        ${optionalString (kcfg.key != null) ''
-          cp ${toString kcfg.key} ${scfg.configDir}/key.pem
-          chown ${scfg.user}:${scfg.group} ${scfg.configDir}/key.pem
-          chmod 400 ${scfg.configDir}/key.pem
-        ''}
-      '';
-    };
-
-    systemd.services.syncthing-init = {
-      after = [ "syncthing.service" ];
-      wantedBy = [ "multi-user.target" ];
-
-      serviceConfig = {
-        User = scfg.user;
-        RemainAfterExit = true;
-        Type = "oneshot";
-        ExecStart = updateConfig;
-      };
-    };
-  };
-}
diff --git a/krebs/5pkgs/haskell/reaktor2.nix b/krebs/5pkgs/haskell/reaktor2.nix
index f06e79121..ae242efea 100644
--- a/krebs/5pkgs/haskell/reaktor2.nix
+++ b/krebs/5pkgs/haskell/reaktor2.nix
@@ -1,17 +1,18 @@
 { mkDerivation, aeson, async, attoparsec, base, blessings
 , bytestring, containers, data-default, fetchgit, filepath
 , hashable, lens, lens-aeson, network, network-simple
-, network-simple-tls, pcre-light, process, random, stdenv
-, string-conversions, stringsearch, text, time, transformers
-, unagi-chan, unix, unordered-containers, vector
+, network-simple-tls, network-uri, pcre-light, process, random
+, servant-server, stdenv, string-conversions, stringsearch, text
+, time, transformers, unagi-chan, unix, unordered-containers
+, vector, wai, warp
 }:
 mkDerivation {
   pname = "reaktor2";
-  version = "0.2.2";
+  version = "0.3.0";
   src = fetchgit {
     url = "https://cgit.krebsco.de/reaktor2";
-    sha256 = "1kyr5i5zdzvc7fcyac1i1yvi88kcxafrgp8p79c1b9l4g9sjnv78";
-    rev = "9f4e2644188f985d7cd806c13e2c0dee1688b9f0";
+    sha256 = "02hqpq8wcfd6rvi8qk10zy3f3lrzzqnjwqal4cbvksjn3vahz36h";
+    rev = "a6893c00f78a8acd0a4bfe7da87ab6889eabcf21";
     fetchSubmodules = true;
   };
   isLibrary = false;
@@ -19,9 +20,9 @@ mkDerivation {
   executableHaskellDepends = [
     aeson async attoparsec base blessings bytestring containers
     data-default filepath hashable lens lens-aeson network
-    network-simple network-simple-tls pcre-light process random
-    string-conversions stringsearch text time transformers unagi-chan
-    unix unordered-containers vector
+    network-simple network-simple-tls network-uri pcre-light process
+    random servant-server string-conversions stringsearch text time
+    transformers unagi-chan unix unordered-containers vector wai warp
   ];
   license = stdenv.lib.licenses.mit;
 }
diff --git a/krebs/5pkgs/simple/grib2json/default.nix b/krebs/5pkgs/simple/grib2json/default.nix
new file mode 100644
index 000000000..2a7959a75
--- /dev/null
+++ b/krebs/5pkgs/simple/grib2json/default.nix
@@ -0,0 +1,19 @@
+{ stdenv, fetchzip, makeWrapper, jre }:
+
+stdenv.mkDerivation {
+  name = "grib2json";
+  src = fetchzip {
+    url = "https://github.com/krebs/grib2json-bin/archive/31efe677b40fe491c988d50f96b59b1b7e2d46f7.zip";
+    sha256 = "1h3yxg270bvac7kaqsbsv4f8nln1i03rfz4cm8cp7llhjj3s6a6b";
+  };
+  installPhase = ''
+    mkdir -p $out/bin
+    cp -r lib $out/
+    cat > $out/bin/grib2json << EOF
+    #!/bin/sh
+    set -x
+    ${jre}/bin/java -jar $out/lib/grib2json-0.8.0-SNAPSHOT.jar "\$@"
+    EOF
+    chmod +x $out/bin/grib2json
+  '';
+}
diff --git a/krebs/5pkgs/simple/irc-announce/default.nix b/krebs/5pkgs/simple/irc-announce/default.nix
index a90814aa9..52cf12862 100644
--- a/krebs/5pkgs/simple/irc-announce/default.nix
+++ b/krebs/5pkgs/simple/irc-announce/default.nix
@@ -25,10 +25,10 @@ pkgs.writeDashBin "irc-announce" ''
   # This is used to see what we send to the irc server. (debug output)
   echo2() { echo "$*"; echo "$*" >&2; }
   cat2() {
-    while read -r line; do
-      echo "$line"
-      echo "$line" >&2
-    done
+    awk '{
+      print $0
+      print $0 > "/dev/stderr"
+    }'
   }
 
   # privmsg_cat transforms stdin to a privmsg
diff --git a/krebs/5pkgs/simple/nomads-cloud/default.nix b/krebs/5pkgs/simple/nomads-cloud/default.nix
new file mode 100644
index 000000000..6e4ace484
--- /dev/null
+++ b/krebs/5pkgs/simple/nomads-cloud/default.nix
@@ -0,0 +1,43 @@
+{ writers, coreutils, grib2json, curl, jq, findutils, imagemagick }:
+writers.writeDashBin "nomads-cloud" ''
+  prefix=$(mktemp -d)
+  grib_path=$prefix.grib
+  json_path=$prefix.json
+  pgm_path=$prefix.pgm
+  png_path="$1"
+
+  mkdir -p "$prefix"
+
+  date=$(${coreutils}/bin/date +%Y%m%d)
+  for hour in 18 12 06 00; do
+    url="https://nomads.ncep.noaa.gov/cgi-bin/filter_gfs_0p25_1hr.pl?file=gfs.t''${hour}z.pgrb2.0p25.anl&lev_entire_atmosphere_%5C%28considered_as_a_single_layer%5C%29=on&var_CWAT=on&leftlon=-180&rightlon=180&toplat=90&bottomlat=-90&dir=%2Fgfs.$date%2F$hour"
+    ${curl}/bin/curl -fsS "$url" > "$grib_path"
+    if [ "$?" -eq 0 ]; then
+      break
+    fi
+  done
+  ${grib2json}/bin/grib2json --data "$grib_path" > "$json_path"
+
+  width=$(${jq}/bin/jq '.[0].header.nx' < "$json_path")
+  height=$(${jq}/bin/jq '.[0].header.ny' < "$json_path")
+
+  # The maximum gray value.  Must be bigger than 0 and less than 65536.
+  maxval=1000
+
+  # pgm - Netpbm grayscale image format
+  # http://netpbm.sourceforge.net/doc/pgm.html
+  {
+    echo P2
+    echo "$width $height"
+    echo "$maxval"
+    cat "$json_path" |
+    ${jq}/bin/jq --argjson maxval "$maxval" -c '
+      ((.[0].data[]) * $maxval | round)
+    ' |
+    ${findutils}/bin/xargs -n "$width"
+  } > "$pgm_path"
+
+  ${imagemagick}/bin/convert -roll +50% "$pgm_path" "$png_path"
+
+  rm -r "$prefix"
+''
diff --git a/krebs/5pkgs/simple/realwallpaper/default.nix b/krebs/5pkgs/simple/realwallpaper/default.nix
index ef83d2e59..057983fec 100644
--- a/krebs/5pkgs/simple/realwallpaper/default.nix
+++ b/krebs/5pkgs/simple/realwallpaper/default.nix
@@ -1,5 +1,5 @@
 { pkgs, ... }:
-pkgs.writeDashBin "generate-wallpaper" ''
+pkgs.writers.writeDashBin "generate-wallpaper" ''
   set -euf
 
   # usage: getimg FILENAME URL
@@ -17,24 +17,28 @@ pkgs.writeDashBin "generate-wallpaper" ''
     fi
   }
 
-  # usage: image_size FILENAME
-  image_size() {
-    identify "$1" | awk '{print$3}'
+  # check if file exists and fetch only if miss