Merge remote-tracking branch 'origin/master'

7 files changed, 71 insertions, 240 deletions

diff --git a/krebs/5pkgs/simple/Reaktor/default.nix b/krebs/5pkgs/simple/Reaktor/default.nix
index 6989bb02b..a9566087f 100644
--- a/krebs/5pkgs/simple/Reaktor/default.nix
+++ b/krebs/5pkgs/simple/Reaktor/default.nix
@@ -1,8 +1,8 @@
-{ lib, pkgs,python3Packages,fetchurl, ... }:
+{ lib, pkgs, python3Packages, fetchFromGitHub, ... }:
 
 python3Packages.buildPythonPackage rec {
   name = "Reaktor-${version}";
-  version = "0.5.1";
+  version = "0.6.0";
 
   doCheck = false;
 
@@ -10,9 +10,11 @@ python3Packages.buildPythonPackage rec {
     python3Packages.docopt
     python3Packages.requests
   ];
-  src = fetchurl {
-    url = "https://pypi.python.org/packages/source/R/Reaktor/Reaktor-${version}.tar.gz";
-    sha256 = "0dn9r0cyxi1sji2pnybsrc4hhaaq7hmf235nlgkrxqlsdb7y6n6n";
+  src = fetchFromGitHub {
+    owner = "krebs";
+    repo = "Reaktor";
+    rev = version;
+    sha256 = "0nsnv1rixmlg5wkb74b4f5bycb42b9rp4b14hijh558hbsa1b9am";
   };
   meta = {
     homepage = http://krebsco.de/;
diff --git a/krebs/5pkgs/simple/Reaktor/plugins.nix b/krebs/5pkgs/simple/Reaktor/plugins.nix
index 92a270ef3..6f59ad1de 100644
--- a/krebs/5pkgs/simple/Reaktor/plugins.nix
+++ b/krebs/5pkgs/simple/Reaktor/plugins.nix
@@ -120,7 +120,7 @@ rec {
   url-title = (buildSimpleReaktorPlugin "url-title" {
     pattern = "^.*(?P<args>http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+).*$$";
     path = with pkgs; [ curl perl ];
-    script = pkgs.writePython3 "url-title" [ "beautifulsoup4" "lxml" ] ''
+    script = pkgs.writePython3 "url-title" { deps = with pkgs.python3Packages; [ beautifulsoup4 lxml ]; } ''
       import cgi
       import sys
       import urllib.request
diff --git a/krebs/5pkgs/simple/Reaktor/scripts/sed-plugin.py b/krebs/5pkgs/simple/Reaktor/scripts/sed-plugin.py
index 51ac7a071..ecb03917b 100644
--- a/krebs/5pkgs/simple/Reaktor/scripts/sed-plugin.py
+++ b/krebs/5pkgs/simple/Reaktor/scripts/sed-plugin.py
@@ -8,9 +8,19 @@ import shelve
 from os import environ
 from os.path import join
 from sys import argv
+from time import sleep
 import re
 
-d = shelve.open(join(environ['state_dir'], 'sed-plugin.shelve'), writeback=True)
+# try to open the shelve file until it succeeds
+while True:
+    try:
+        d = shelve.open(
+            join(environ['state_dir'], 'sed-plugin.shelve'),
+            writeback=True
+        )
+        break
+    except:  # noqa: E722
+        sleep(0.2)
 usr = environ['_from']
 
 
diff --git a/krebs/5pkgs/simple/buildbot-classic/default.nix b/krebs/5pkgs/simple/buildbot-classic/default.nix
index 4fae6256d..665b36ab4 100644
--- a/krebs/5pkgs/simple/buildbot-classic/default.nix
+++ b/krebs/5pkgs/simple/buildbot-classic/default.nix
@@ -2,7 +2,7 @@
 
 python2Packages.buildPythonApplication rec {
   name = "buildbot-classic-${version}";
-  version = "0.8.17";
+  version = "0.8.18";
   namePrefix = "";
   patches = [];
 
@@ -10,7 +10,7 @@ python2Packages.buildPythonApplication rec {
     owner = "krebs";
     repo = "buildbot-classic";
     rev = version;
-    sha256 = "0yn0n37rs2bhz9q0simnvyzz5sfrpqhbdm6pdj6qk7sab4y6xbq8";
+    sha256 = "0b4y3n9zd2gdy8xwk1vpvs4n9fbg72vi8mx4ydgijwngcmdqkjmq";
   };
   postUnpack = "sourceRoot=\${sourceRoot}/master";
 
diff --git a/krebs/5pkgs/simple/krebspaste/default.nix b/krebs/5pkgs/simple/krebspaste/default.nix
index 31ad12780..a11c8c90a 100644
--- a/krebs/5pkgs/simple/krebspaste/default.nix
+++ b/krebs/5pkgs/simple/krebspaste/default.nix
@@ -1,6 +1,6 @@
 { writeDashBin, bepasty-client-cli }:
 
-# TODO use `execve` instead?
+# TODO use `pkgs.exec` instead?
 writeDashBin "krebspaste" ''
   exec ${bepasty-client-cli}/bin/bepasty-cli -L 1m --url http://paste.r "$@" | sed '$ s/$/\/+inline/g'
 ''
diff --git a/krebs/5pkgs/simple/stockholm/default.nix b/krebs/5pkgs/simple/stockholm/default.nix
deleted file mode 100644
index c973386d6..000000000
--- a/krebs/5pkgs/simple/stockholm/default.nix
+++ /dev/null
@@ -1,230 +0,0 @@
-{ pkgs }: let
-
-  stockholm-dir = ../../../..;
-
-  lib = import (stockholm-dir + "/lib");
-
-  #
-  # high level commands
-  #
-
-  cmds.deploy = pkgs.withGetopt {
-    force-populate = { default = /* sh */ "false"; switch = true; };
-    quiet = { default = /* sh */ "false"; switch = true; };
-    source_file = {
-      default = /* sh */ "$user/1systems/$system/source.nix";
-      long = "source";
-    };
-    system = {};
-    target.default = /* sh */ "$system";
-    user.default = /* sh */ "$LOGNAME";
-  } (opts: pkgs.writeDash "stockholm.deploy" ''
-    set -efu
-
-    . ${init.env}
-    . ${init.proxy "deploy" opts}
-
-    # Use system's nixos-rebuild, which is not self-contained
-    export PATH=/run/current-system/sw/bin
-    exec ${utils.with-whatsupnix} \
-    nixos-rebuild switch \
-        --show-trace \
-        -I "$target_path"
-  '');
-
-  cmds.get-version = pkgs.writeDash "get-version" ''
-    set -efu
-    hostname=''${HOSTNAME-$(${pkgs.nettools}/bin/hostname)}
-    version=git.$(${pkgs.git}/bin/git describe --always --dirty)
-    case $version in (*-dirty)
-      version=$version@$hostname
-    esac
-    date=$(${pkgs.coreutils}/bin/date +%y.%m)
-    echo "$date.$version"
-  '';
-
-  cmds.install = pkgs.withGetopt {
-    force-populate = { default = /* sh */ "false"; switch = true; };
-    quiet = { default = /* sh */ "false"; switch = true; };
-    source_file = {
-      default = /* sh */ "$user/1systems/$system/source.nix";
-      long = "source";
-    };
-    system = {};
-    target = {};
-    user.default = /* sh */ "$LOGNAME";
-  } (opts: pkgs.writeBash "stockholm.install" ''
-    set -efu
-
-    . ${init.env}
-
-    if \test "''${using_proxy-}" != true; then
-      ${pkgs.openssh}/bin/ssh \
-          -o StrictHostKeyChecking=no \
-          -o UserKnownHostsFile=/dev/null \
-          "$target_user@$target_host" -p "$target_port" \
-          env target_path=$(${pkgs.quote}/bin/quote "$target_path") \
-              sh -s prepare \
-            < ${stockholm-dir + "/krebs/4lib/infest/prepare.sh"}
-              # TODO inline prepare.sh?
-    fi
-
-    . ${init.proxy "install" opts}
-
-    # these variables get defined by nix-shell (i.e. nix-build) from
-    # XDG_RUNTIME_DIR and reference the wrong directory (/run/user/0),
-    # which only exists on / and not at /mnt.
-    export NIX_BUILD_TOP=/tmp
-    export TEMPDIR=/tmp
-    export TEMP=/tmp
-    export TMPDIR=/tmp
-    export TMP=/tmp
-    export XDG_RUNTIME_DIR=/tmp
-
-    export NIXOS_CONFIG="$target_path/nixos-config"
-
-    cd
-    exec nixos-install
-  '');
-
-  cmds.test = pkgs.withGetopt {
-    force-populate = { default = /* sh */ "false"; switch = true; };
-    quiet = { default = /* sh */ "false"; switch = true; };
-    source_file = {
-      default = /* sh */ "$user/1systems/$system/source.nix";
-      long = "source";
-    };
-    system = {};
-    target = {};
-    user.default = /* sh */ "$LOGNAME";
-  } (opts: pkgs.writeDash "stockholm.test" /* sh */ ''
-    set -efu
-
-    export dummy_secrets=true
-
-    . ${init.env}
-    . ${init.proxy "test" opts}
-
-    exec ${utils.build} config.system.build.toplevel
-  '');
-
-  #
-  # low level commands
-  #
-
-  # usage: get-source SOURCE_FILE
-  cmds.get-source = pkgs.writeDash "stockholm.get-source" ''
-    set -efu
-    exec ${pkgs.nix}/bin/nix-instantiate \
-        --eval \
-        --json \
-        --readonly-mode \
-        --show-trace \
-        --strict \
-        "$1"
-  '';
-
-  # usage: parse-target [--default=TARGET] TARGET
-  # TARGET = [USER@]HOST[:PORT][/PATH]
-  cmds.parse-target = pkgs.withGetopt {
-    default_target = {
-      long = "default";
-      short = "d";
-    };
-  } (opts: pkgs.writeDash "stockholm.parse-target" ''
-    set -efu
-    target=$1; shift
-    for arg; do echo "$0: bad argument: $arg" >&2; done
-    if \test $# != 0; then exit 2; fi
-    exec ${pkgs.jq}/bin/jq \
-        -enr \
-        --arg default_target "$default_target" \
-        --arg target "$target" \
-        -f ${pkgs.writeText "stockholm.parse-target.jq" ''
-          def parse: match("^(?:([^@]+)@)?([^:/]+)?(?::([0-9]+))?(/.*)?$") | {
-            user: .captures[0].string,
-            host: .captures[1].string,
-            port: .captures[2].string,
-            path: .captures[3].string,
-          };
-          def sanitize: with_entries(select(.value != null));
-          ($default_target | parse) + ($target | parse | sanitize) |
-          . + { local: (.user == env.LOGNAME and .host == env.HOSTNAME) }
-        ''}
-  '');
-
-  init.env = pkgs.writeText "init.env" /* sh */ ''
-
-    export HOSTNAME="$(${pkgs.nettools}/bin/hostname)"
-
-    export quiet
-    export system
-    export target
-    export user
-
-    default_target=root@$system:22/var/src
-
-    export target_object="$(
-      ${cmds.parse-target} "$target" -d "$default_target"
-    )"
-    export target_user="$(echo $target_object | ${pkgs.jq}/bin/jq -r .user)"
-    export target_host="$(echo $target_object | ${pkgs.jq}/bin/jq -r .host)"
-    export target_port="$(echo $target_object | ${pkgs.jq}/bin/jq -r .port)"
-    export target_path="$(echo $target_object | ${pkgs.jq}/bin/jq -r .path)"
-    export target_local="$(echo $target_object | ${pkgs.jq}/bin/jq -r .local)"
-  '';
-
-  init.proxy = command: opts: pkgs.writeText "init.proxy" /* sh */ ''
-    if \test "''${using_proxy-}" != true; then
-
-      source=$(${cmds.get-source} "$source_file")
-      qualified_target=$target_user@$target_host:$target_port$target_path
-      if \test "$force_populate" = true; then
-        echo "$source" | ${pkgs.populate}/bin/populate --force "$qualified_target"
-      else
-        echo "$source" | ${pkgs.populate}/bin/populate "$qualified_target"
-      fi
-
-      if \test "$target_local" != true; then
-        exec ${pkgs.openssh}/bin/ssh \
-            "$target_user@$target_host" -p "$target_port" \
-            cd "$target_path/stockholm" \; \
-            NIX_PATH=$(${pkgs.quote}/bin/quote "$target_path") \
-            nix-shell --run "$(${pkgs.quote}/bin/quote "
-              ${lib.concatStringsSep " " (lib.mapAttrsToList
-                (name: opt: /* sh */
-                  "${opt.varname}=\$(${pkgs.quote}/bin/quote ${opt.ref})")
-                opts
-              )} \
-              using_proxy=true \
-              ${lib.shell.escape command} \
-              $WITHGETOPT_ORIG_ARGS \
-            ")"
-      fi
-    fi
-  '';
-
-  utils.build = pkgs.writeDash "utils.build" ''
-    set -efu
-    ${utils.with-whatsupnix} \
-    ${pkgs.nix}/bin/nix-build \
-        --no-out-link \
-        --show-trace \
-        -E "with import <stockholm>; $1" \
-        -I "$target_path" \
-  '';
-
-  utils.with-whatsupnix = pkgs.writeDash "utils.with-whatsupnix" ''
-    set -efu
-    if \test "$quiet" = true; then
-      "$@" -Q 2>&1 | ${pkgs.whatsupnix}/bin/whatsupnix
-    else
-      exec "$@"
-    fi
-  '';
-
-in
-
-  pkgs.write "stockholm" (lib.mapAttrs' (name: link:
-    lib.nameValuePair "/bin/${name}" { inherit link; }
-  ) cmds)
diff --git a/krebs/5pkgs/simple/syncthing-device-id.nix b/krebs/5pkgs/simple/syncthing-device-id.nix
new file mode 100644
index 000000000..9533800fd
--- /dev/null
+++ b/krebs/5pkgs/simple/syncthing-device-id.nix
@@ -0,0 +1,49 @@
+{ openssl, writePython2Bin }:
+
+writePython2Bin "syncthing-device-id" {
+  flakeIgnore = [
+    "E226"
+    "E302"
+    "E305"
+    "E501"
+    "F401"
+  ];
+} /* python */ ''
+  import base64
+  import hashlib
+  import subprocess
+  import sys
+
+  B32ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
+
+  def luhn_checksum(data, alphabet=B32ALPHABET):
+      n = len(alphabet)
+      number = tuple(alphabet.index(i) for i in reversed(data))
+      result = (sum(number[::2]) +
+                sum(sum(divmod(i * 2, n)) for i in number[1::2])) % n
+      return alphabet[-result]
+
+  def main(incert):
+      der_data = subprocess.check_output([
+          '${openssl}/bin/openssl',
+          'x509',
+          '-outform',
+          'DER',
+      ], stdin=incert)
+      data_hash = hashlib.sha256(der_data)
+      b32_hash = base64.b32encode(data_hash.digest()).decode('ascii')
+
+      result = b32_hash.upper().rstrip('=')
+      blocks = [result[pos:pos+13] for pos in range(0, len(result), 13)]
+      result = '''.join(block + luhn_checksum(block) for block in blocks)
+
+      blocks = [result[pos:pos+7] for pos in range(0, len(result), 7)]
+      print('-'.join(blocks))
+
+  if __name__ == '__main__':
+      import argparse
+      parser = argparse.ArgumentParser(description='Generate syncthing ID from certificate')
+      parser.add_argument('incert', type=argparse.FileType('rb'), help='Certificate path')
+      args = parser.parse_args()
+      main(**vars(args))
+''