Merge remote-tracking branch 'lass/master'

55 files changed, 65 insertions, 755 deletions

diff --git a/default.nix b/default.nix
index cab55d40a..5ae8e399e 100644
--- a/default.nix
+++ b/default.nix
@@ -13,10 +13,7 @@ import <nixpkgs/nixos/lib/eval-config.nix> {
       (attrNames (filterAttrs (_: eq "directory") (readDir (<stockholm> + "/${ns}/1systems"))))
       (name: let
         config = import (<stockholm> + "/${ns}/1systems/${name}/config.nix");
-        source = import (<stockholm> + "/${ns}/1systems/${name}/source.nix");
       in import <nixpkgs/nixos/lib/eval-config.nix> {
         modules = [ config ];
-      } // {
-        inherit source;
       });
 }
diff --git a/jeschli/1systems/bln/source.nix b/jeschli/1systems/bln/source.nix
deleted file mode 100644
index 0864fd90c..000000000
--- a/jeschli/1systems/bln/source.nix
+++ /dev/null
@@ -1,4 +0,0 @@
-import <stockholm/jeschli/source.nix> {
-  name = "bln";
-  secure = true;
-}
diff --git a/jeschli/1systems/bolide/source.nix b/jeschli/1systems/bolide/source.nix
deleted file mode 100644
index 0bd7af50f..000000000
--- a/jeschli/1systems/bolide/source.nix
+++ /dev/null
@@ -1,4 +0,0 @@
-import <stockholm/jeschli/source.nix> {
-  name = "bolide";
-  secure = true;
-}
diff --git a/jeschli/1systems/brauerei/source.nix b/jeschli/1systems/brauerei/source.nix
deleted file mode 100644
index 61978768e..000000000
--- a/jeschli/1systems/brauerei/source.nix
+++ /dev/null
@@ -1,4 +0,0 @@
-import <stockholm/jeschli/source.nix> {
-  name = "brauerei";
-  secure = true;
-}
diff --git a/jeschli/1systems/enklave/source.nix b/jeschli/1systems/enklave/source.nix
deleted file mode 100644
index 4f9f37be7..000000000
--- a/jeschli/1systems/enklave/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/jeschli/source.nix> {
-  name = "enklave";
-}
diff --git a/jeschli/1systems/reagenzglas/.source.nix.swp b/jeschli/1systems/reagenzglas/.source.nix.swp
deleted file mode 100644
index 8c1a75f39..000000000
--- a/jeschli/1systems/reagenzglas/.source.nix.swp
+++ /dev/null
diff --git a/jeschli/1systems/reagenzglas/source.nix b/jeschli/1systems/reagenzglas/source.nix
deleted file mode 100644
index 7543de6b9..000000000
--- a/jeschli/1systems/reagenzglas/source.nix
+++ /dev/null
@@ -1,4 +0,0 @@
-import <stockholm/jeschli/source.nix> {
-  name = "reagenzglas";
-  secure = true;
-}
diff --git a/jeschli/source.nix b/jeschli/source.nix
deleted file mode 100644
index fc1413ee4..000000000
--- a/jeschli/source.nix
+++ /dev/null
@@ -1,26 +0,0 @@
-with import <stockholm/lib>;
-host@{ name, secure ? false, override ? {} }: let
-  builder = if getEnv "dummy_secrets" == "true"
-              then "buildbot"
-              else "jeschli";
-  _file = <stockholm> + "/jeschli/1systems/${name}/source.nix";
-  pkgs = import <nixpkgs> {
-    overlays = map import [
-      <stockholm/krebs/5pkgs>
-      <stockholm/submodules/nix-writers/pkgs>
-    ];
-  };
-in
-  evalSource (toString _file) [
-    {
-      nixos-config.symlink = "stockholm/jeschli/1systems/${name}/config.nix";
-      nixpkgs = (import <stockholm/krebs/source.nix> host).nixpkgs;
-      secrets.file = getAttr builder {
-        buildbot = toString <stockholm/jeschli/2configs/tests/dummy-secrets>;
-        jeschli = "${getEnv "HOME"}/secrets/${name}";
-      };
-      stockholm.file = toString <stockholm>;
-      stockholm-version.pipe = "${pkgs.stockholm}/bin/get-version";
-    }
-    override
-  ]
diff --git a/krebs/0tests/deploy.nix b/krebs/0tests/deploy.nix
index d96963500..5fae60ecc 100644
--- a/krebs/0tests/deploy.nix
+++ b/krebs/0tests/deploy.nix
@@ -44,11 +44,6 @@ let
     exec >&2
     source=${pkgs.writeJSON "source.json" populate-source}
     LOGNAME=krebs ${pkgs.populate}/bin/populate --force root@server:22/var/src/ < "$source"
-    # TODO: make deploy work
-    #LOGNAME=krebs ${pkgs.stockholm}/bin/deploy \
-    #    --force-populate \
-    #    --source=${./data/test-source.nix} \
-    #    --system=server \
   '';
   minimalSystem = (import <nixpkgs/nixos/lib/eval-config.nix> {
     modules = [
diff --git a/krebs/1systems/hotdog/source.nix b/krebs/1systems/hotdog/source.nix
deleted file mode 100644
index 0fa61b20f..000000000
--- a/krebs/1systems/hotdog/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "hotdog";
-}
diff --git a/krebs/1systems/onebutton/source.nix b/krebs/1systems/onebutton/source.nix
deleted file mode 100644
index 91a998de7..000000000
--- a/krebs/1systems/onebutton/source.nix
+++ /dev/null
@@ -1,13 +0,0 @@
-with import <stockholm/lib>;
-let
-  pkgs = import <nixpkgs> {};
-  nixpkgs = builtins.fetchTarball {
-    url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz;
-  };
-in import <stockholm/krebs/source.nix> {
-  name = "onebutton";
-  override.nixpkgs = mkForce {
-    file = toString nixpkgs;
-  };
-
-}
diff --git a/krebs/1systems/puyak/source.nix b/krebs/1systems/puyak/source.nix
deleted file mode 100644
index a21651899..000000000
--- a/krebs/1systems/puyak/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "puyak";
-}
diff --git a/krebs/1systems/test-all-krebs-modules/source.nix b/krebs/1systems/test-all-krebs-modules/source.nix
deleted file mode 100644
index 66fdaa773..000000000
--- a/krebs/1systems/test-all-krebs-modules/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "test-all-krebs-modules";
-}
diff --git a/krebs/1systems/test-arch/source.nix b/krebs/1systems/test-arch/source.nix
deleted file mode 100644
index bff9d4325..000000000
--- a/krebs/1systems/test-arch/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "test-arch";
-}
diff --git a/krebs/1systems/test-centos6/source.nix b/krebs/1systems/test-centos6/source.nix
deleted file mode 100644
index 3693bbb29..000000000
--- a/krebs/1systems/test-centos6/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "test-centos6";
-}
diff --git a/krebs/1systems/test-centos7/source.nix b/krebs/1systems/test-centos7/source.nix
deleted file mode 100644
index 44230f08d..000000000
--- a/krebs/1systems/test-centos7/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "test-centos7";
-}
diff --git a/krebs/1systems/test-failing/source.nix b/krebs/1systems/test-failing/source.nix
deleted file mode 100644
index 60b77a0a0..000000000
--- a/krebs/1systems/test-failing/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "test-failing";
-}
diff --git a/krebs/1systems/test-minimal-deploy/source.nix b/krebs/1systems/test-minimal-deploy/source.nix
deleted file mode 100644
index 032ab12bb..000000000
--- a/krebs/1systems/test-minimal-deploy/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "test-minimal-deploy";
-}
diff --git a/krebs/1systems/wolf/source.nix b/krebs/1systems/wolf/source.nix
deleted file mode 100644
index c292bfa62..000000000
--- a/krebs/1systems/wolf/source.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-import <stockholm/krebs/source.nix> {
-  name = "wolf";
-}
diff --git a/krebs/2configs/buildbot-stockholm.nix b/krebs/2configs/buildbot-stockholm.nix
index 0aa9288ec..0ee91ae34 100644
--- a/krebs/2configs/buildbot-stockholm.nix
+++ b/krebs/2configs/buildbot-stockholm.nix
@@ -1,47 +1,11 @@
-{ config, pkgs, ... }: with import <stockholm/lib>;
+{ config, ... }: with import <stockholm/lib>;
 
-let
-
-  hostname = config.networking.hostName;
-
-  sourceRepos = [
-    "http://cgit.enklave.r/stockholm"
-    "http://cgit.gum.r/stockholm"
-    "http://cgit.hotdog.r/stockholm"
-    "http://cgit.ni.r/stockholm"
-    "http://cgit.prism.r/stockholm"
-  ];
-
-  # usage: build USER HOST
-  # This executable is meant to be run with <stockholm> as working directory.
-  # USER is expected to be a subdirectory of the working directory.
-  build = pkgs.writeDash "build" ''
-    set -efu
-
-    user=$1
-    host=$2
-
-    result=$(nix-build \
-        --argstr name "$host" \
-        --argstr target "$HOME"/stockholm-build \
-        --attr test \
-        --no-build-output \
-        --no-out-link \
-        --show-trace \
-        "$user"/krops.nix \
-    )
-
-    exec "$result"
-  '';
-
-
-in
 {
   networking.firewall.allowedTCPPorts = [ 80 ];
   services.nginx = {
     enable = true;
     virtualHosts.build = {
-      serverAliases = [ "build.${hostname}.r" ];
+      serverAliases = [ "build.${config.networking.hostName}.r" ];
       locations."/".extraConfig = ''
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "upgrade";
@@ -49,155 +13,16 @@ in
       '';
     };
   };
-
-  krebs.buildbot.master = {
-    slaves = {
-      testslave = "lasspass";
-    };
-    change_source.stockholm = concatMapStrings (repo: ''
-      cs.append(
-          changes.GitPoller(
-              "${repo}",
-              workdir='stockholm${elemAt(splitString "." repo) 1}', branches=True,
-              project='stockholm',
-              pollinterval=10
-          )
-      )
-    '') sourceRepos;
-    scheduler = {
-      auto-scheduler = ''
-        sched.append(
-              schedulers.SingleBranchScheduler(
-                  change_filter=util.ChangeFilter(branch_re=".*"),
-                  treeStableTimer=60,
-                  name="build-all-branches",
-                  builderNames=[
-                      "hosts",
-                  ]
-              )
-        )
-      '';
-      force-scheduler = ''
-        sched.append(
-            schedulers.ForceScheduler(
-                  name="hosts",
-                  builderNames=[
-                      "hosts",
-                  ]
-            )
-        )
-      '';
-    };
-    builder_pre = ''
-      # prepare grab_repo step for stockholm
-      grab_repo = steps.Git(
-          repourl=util.Property('repository', 'http://cgit.hotdog.r/stockholm'),
-          mode='full',
-          submodules=True,
-      )
-    '';
-    builder = {
-      hosts = ''
-        from buildbot import interfaces
-        from buildbot.steps.shell import ShellCommand
-
-        class StepToStartMoreSteps(ShellCommand):
-            def __init__(self, **kwargs):
-                ShellCommand.__init__(self, **kwargs)
-
-            def addBuildSteps(self, steps_factories):
-                for sf in steps_factories:
-                    step = interfaces.IBuildStepFactory(sf).buildStep()
-                    step.setBuild(self.build)
-                    step.setBuildSlave(self.build.slavebuilder.slave)
-                    step_status = self.build.build_status.addStepWithName(step.name)
-                    step.setStepStatus(step_status)
-                    self.build.steps.append(step)
-
-            def start(self):
-                props = self.build.getProperties()
-                hosts = json.loads(props.getProperty('hosts_json'))
-                for host in hosts:
-                    user = hosts[host]['owner']
-
-                    self.addBuildSteps([steps.ShellCommand(
-                        name=str(host),
-                        env={
-                          "NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src",
-                          "NIX_REMOTE": "daemon",
-                        },
-                        command=[
-                          "${build}", user, host
-                        ],
-                        timeout=90001,
-                        workdir='build', # TODO figure out why we need this?
-                    )])
-
-                ShellCommand.start(self)
-
-
-        f = util.BuildFactory()
-        f.addStep(grab_repo)
-
-        f.addStep(steps.SetPropertyFromCommand(
-            env={
-              "NIX_PATH": "secrets=/var/src/stockholm/null:stockholm=./:/var/src",
-              "NIX_REMOTE": "daemon",
-            },
-            name="get_hosts",
-            command=["nix-instantiate", "--json", "--strict", "--eval", "-E", """
-                with import <nixpkgs> {};
-                let
-                  eval-config = cfg:
-                    import <nixpkgs/nixos/lib/eval-config.nix> {
-                      modules = [
-                        (import cfg)
-                      ];
-                    }
-                  ;
-
-                  system = eval-config ./krebs/1systems/hotdog/config.nix; # TODO put a better config here
-
-                  ci-systems = lib.filterAttrs (_: v: v.ci) system.config.krebs.hosts;
-
-                  filtered-attrs = lib.mapAttrs ( n: v: {
-                    owner = v.owner.name;
-                  }) ci-systems;
-
-                in filtered-attrs
-            """],
-            property="hosts_json"
-        ))
-        f.addStep(StepToStartMoreSteps(command=["echo"])) # TODO remove dummy command from here
-
-        bu.append(
-            util.BuilderConfig(
-                name="hosts",
-                slavenames=slavenames,
-                factory=f
-            )
-        )
-      '';
-    };
+  krebs.ci = {
     enable = true;
-    web.enable = true;
-    irc = {
-      enable = true;
-      nick = "build|${hostname}";
-      server = "irc.r";
-      channels = [ "noise" "xxx" ];
-      allowForce = true;
+    repos = {
+      stockholm.urls = [
+        "http://cgit.enklave.r/stockholm"
+        "http://cgit.gum.r/stockholm"
+        "http://cgit.hotdog.r/stockholm"
+        "http://cgit.ni.r/stockholm"
+        "http://cgit.prism.r/stockholm"
+      ];
     };
-    extraConfig = ''
-      c['buildbotURL'] = "http://build.${hostname}.r/"
-    '';
-  };
-
-  krebs.buildbot.slave = {
-    enable = true;
-    masterhost = "localhost";
-    username = "testslave";
-    password = "lasspass";
-    packages = with pkgs; [ gnumake jq nix populate gnutar lzma gzip ];
   };
 }
diff --git a/krebs/2configs/default.nix b/krebs/2configs/default.nix
index 7b970923d..fafcd72c3 100644
--- a/krebs/2configs/default.nix
+++ b/krebs/2configs/default.nix
@@ -49,6 +49,7 @@ with import <stockholm/lib>;
   users.mutableUsers = false;
   users.extraUsers.root.openssh.authorizedKeys.keys = [
     # TODO
+    config.krebs.users.jeschli-brauerei.pubkey
     config.krebs.users.lass.pubkey
     config.krebs.users.lass-mors.pubkey
     config.krebs.users.makefu.pubkey
diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix
index cf82401d3..33afb2c0a 100644
--- a/krebs/3modules/ci.nix
+++ b/krebs/3modules/ci.nix
@@ -141,7 +141,7 @@ let
         enable = true;
         nick = "build|${hostname}";
         server = "irc.r";
-        channels = [ "noise" ];
+        channels = [ "xxx" "noise" ];
         allowForce = true;
       };
       extraConfig = ''
diff --git a/krebs/5pkgs/simple/Reaktor/plugins.nix b/krebs/5pkgs/simple/Reaktor/plugins.nix
index 700f9b40d..92a270ef3 100644
--- a/krebs/5pkgs/simple/Reaktor/plugins.nix
+++ b/krebs/5pkgs/simple/Reaktor/plugins.nix
@@ -160,7 +160,7 @@ rec {
   task-list = buildSimpleReaktorPlugin "task-list" {
     pattern = "^task-list";
     script = pkgs.writeDash "task-list" ''
-        ${pkgs.taskwarrior}/bin/task rc:${taskrcFile} minimal
+        ${pkgs.taskwarrior}/bin/task rc:${taskrcFile} export | ${pkgs.jq}/bin/jq -r '.[] | select(.id != 0) | "\(.id) \(.description)"'
       '';
   };
 
diff --git a/krebs/5pkgs/simple/stockholm/default.nix b/krebs/5pkgs/simple/stockholm/default.nix
deleted file mode 100644
index c973386d6..000000000
--- a/krebs/5pkgs/simple/stockholm/default.nix
+++ /dev/null
@@ -1,230 +0,0 @@
-{ pkgs }: let
-
-  stockholm-dir = ../../../..;
-
-  lib = import (stockholm-dir + "/lib");
-
-  #
-  # high level commands
-  #
-
-  cmds.deploy = pkgs.withGetopt {
-    force-populate = { default = /* sh */ "false"; switch = true; };
-    quiet = { default = /* sh */ "false"; switch = true; };
-    source_file = {
-      default = /* sh */ "$user/1systems/$system/source.nix";
-      long = "source";
-    };
-    system = {};
-    target.default = /* sh */ "$system";
-    user.default = /* sh */ "$LOGNAME";
-  } (opts: pkgs.writeDash "stockholm.deploy" ''
-    set -efu
-
-    . ${init.env}
-    . ${init.proxy "deploy" opts}
-
-    # Use system's nixos-rebuild, which is not self-contained
-    export PATH=/run/current-system/sw/bin
-    exec ${utils.with-whatsupnix} \
-    nixos-rebuild switch \
-        --show-trace \
-        -I "$target_path"
-  '');
-
-  cmds.get-version = pkgs.writeDash "get-version" ''
-    set -efu
-    hostname=''${HOSTNAME-$(${pkgs.nettools}/bin/hostname)}
-    version=git.$(${pkgs.git}/bin/git describe --always --dirty)
-    case $version in (*-dirty)
-      version=$version@$hostname
-    esac
-    date=$(${pkgs.coreutils}/bin/date +%y.%m)
-    echo "$date.$version"
-  '';
-
-  cmds.install = pkgs.withGetopt {
-    force-populate = { default = /* sh */ "false"; switch = true; };
-    quiet = { default = /* sh */ "false"; switch = true; };
-    source_file = {
-      default = /* sh */ "$user/1systems/$system/source.nix";
-      long = "source";
-    };
-    system = {};
-    target = {};
-    user.default = /* sh */ "$LOGNAME";
-  } (opts: pkgs.writeBash "stockholm.install" ''
-    set -efu
-
-    . ${init.env}
-
-    if \test "''${using_proxy-}" != true; then
-      ${pkgs.openssh}/bin/ssh \
-          -o StrictHostKeyChecking=no \
-          -o UserKnownHostsFile=/dev/null \
-          "$target_user@$target_host" -p "$target_port" \
-          env target_path=$(${pkgs.quote}/bin/quote "$target_path") \
-              sh -s prepare \
-            < ${stockholm-dir + "/krebs/4lib/infest/prepare.sh"}
-              # TODO inline prepare.sh?
-    fi
-
-    . ${init.proxy "install" opts}
-
-    # these variables get defined by nix-shell (i.e. nix-build) from
-    # XDG_RUNTIME_DIR and reference the wrong directory (/run/user/0),
-    # which only exists on / and not at /mnt.
-    export NIX_BUILD_TOP=/tmp
-    export TEMPDIR=/tmp
-    export TEMP=/tmp
-    export TMPDIR=/tmp
-    export TMP=/tmp
-    export XDG_RUNTIME_DIR=/tmp
-
-    export NIXOS_CONFIG="$target_path/nixos-config"
-
-    cd
-    exec nixos-install
-  '');
-
-  cmds.test = pkgs.withGetopt {
-    force-populate = { default = /* sh */ "false"; switch = true; };
-    quiet = { default = /* sh */ "false"; switch = true; };
-    source_file = {
-      default = /* sh */ "$user/1systems/$system/source.nix";
-      long = "source";
-    };
-    system = {};
-    target = {};
-    user.default = /* sh */ "$LOGNAME";
-  } (opts: pkgs.writeDash "stockholm.test" /* sh */ ''
-    set -efu
-
-    export dummy_secrets=true
-
-    . ${init.env}
-    . ${init.proxy "test" opts}
-
-    exec ${utils.build} config.system.build.toplevel
-  '');
-
-  #
-  # low level commands
-  #
-
-  # usage: get-source SOURCE_FILE
-  cmds.get-source = pkgs.writeDash "stockholm.get-source" ''
-    set -efu
-    exec ${pkgs.nix}/bin/nix-instantiate \
-        --eval \
-        --json \
-        --readonly-mode \
-        --show-trace \
-        --strict \
-        "$1"
-  '';
-
-  # usage: parse-target [--default=TARGET] TARGET
-  # TARGET = [USER@]HOST[:PORT][/PATH]
-  cmds.parse-target = pkgs.withGetopt {
-    default_target = {
-      long = "default";