Merge remote-tracking branch 'gum/master'

12 files changed, 267 insertions, 105 deletions

diff --git a/krebs/5pkgs/snapraid/default.nix b/krebs/5pkgs/snapraid/default.nix
deleted file mode 100644
index 41db0f284..000000000
--- a/krebs/5pkgs/snapraid/default.nix
+++ /dev/null
@@ -1,33 +0,0 @@
-{stdenv, fetchurl}:
-let
-  s = # Generated upstream information
-  rec {
-    baseName="jq";
-    version="1.5";
-    name="${baseName}-${version}";
-    url=https://github.com/stedolan/jq/releases/download/jq-1.5/jq-1.5.tar.gz;
-    sha256="0g29kyz4ykasdcrb0zmbrp2jqs9kv1wz9swx849i2d1ncknbzln4";
-  };
-  buildInputs = [
-  ];
-in
-stdenv.mkDerivation {
-  inherit (s) name version;
-  inherit buildInputs;
-  src = fetchurl {
-    inherit (s) url sha256;
-  };
-
-  # jq is linked to libjq:
-  configureFlags = [
-    "LDFLAGS=-Wl,-rpath,\\\${libdir}"
-  ];
-  meta = {
-    inherit (s) version;
-    description = ''A lightweight and flexible command-line JSON processor'';
-    license = stdenv.lib.licenses.mit ;
-    maintainers = [stdenv.lib.maintainers.raskin];
-    platforms = stdenv.lib.platforms.linux ++ stdenv.lib.platforms.darwin;
-  };
-}
-
diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes
index eee0bfc34..6bfb6906e 100755
--- a/krebs/5pkgs/test/infest-cac-centos7/notes
+++ b/krebs/5pkgs/test/infest-cac-centos7/notes
@@ -55,9 +55,16 @@ old_trapstr=$(clear_defer)
 while true;do
   # Template 26: CentOS7
   # TODO: use cac templates to determine the real Centos7 template in case it changes
-  name=$( cac build cpu=1 ram=512 storage=10 os=26 2>&1\
-    | jq -r .servername)
-  id=servername:$name
+  out=$(cac build cpu=1 ram=512 storage=10 os=26 2>&1)
+  if name=$(echo "$out" | jq -r .servername);then
+    id=servername:$name
+    echo "got a working machine, id=$id"
+  else
+    echo "Unable to build a virtual machine, retrying in 15 seconds" >&2
+    echo "Output of build program: $out" >&2
+    sleep 15
+    continue
+  fi
 
   clear_defer >/dev/null
   defer "cac delete $id"
@@ -65,8 +72,8 @@ while true;do
   # TODO: timeout?
 
   wait_login_cac(){
-    # we wait for 15 minutes
-    for t in `seq 90`;do
+    # we wait for 30 minutes
+    for t in `seq 180`;do
       # now we have a working cac server
       if cac ssh $1 -o ConnectTimeout=10 \
                     cat /etc/redhat-release | \
@@ -82,6 +89,7 @@ while true;do
     echo "unable to boot a working system within time frame, retrying..." >&2
     echo "Cleaning up old image,last status: $(cac update;cac getserver $id | jq -r .status)"
     eval "$(clear_defer | sed 's/;exit//')"
+    sleep 15
   else
     echo "got a working system" >&2
     break
diff --git a/krebs/default.nix b/krebs/default.nix
index 81ddd3ea6..15d0e8e2e 100644
--- a/krebs/default.nix
+++ b/krebs/default.nix
@@ -50,7 +50,6 @@ let out = {
 
       # Prepare target source via bind-mounting
 
-      (${populate (args // { infesting = true;}) })
 
       (${nixos-install args})
 
@@ -103,6 +102,7 @@ let out = {
       #! /bin/sh
       # ${current-date} ${current-user-name}@${current-host-name}
       # krebs.nixos-install
+      (${populate (args // { root = "/mnt"; })})
 
       ${rootssh target ''
         export PATH; PATH=/root/.nix-profile/bin:$PATH
@@ -209,7 +209,7 @@ let out = {
   populate =
     { system ? current-host-name
     , target ? system
-    , infesting ? false
+    , root ? ""
     }@args:
     let out = ''
         #! /bin/sh
@@ -223,7 +223,6 @@ let out = {
       '';
 
 
-      target_prefix=lib.optionalString infesting "/mnt";
       config = get-config system;
 
       current-host = config.krebs.hosts.${current-host-name};
@@ -232,7 +231,7 @@ let out = {
       methods.dir = config:
         let
           can-push = config.host.name == current-host.name;
-          target-path = target_prefix + config.target-path;
+          target-path = root + config.target-path;
           push-method = ''
             rsync \
               --exclude .git \
@@ -252,7 +251,7 @@ let out = {
         throw "No way to push ${dir} from ${current-host.name} to ${target}";
 
       methods.git = config:
-        let target-path = target_prefix + config.target-path;
+        let target-path = root + config.target-path;
         in rootssh target ''
           mkdir -p ${target-path}
           cd ${target-path}
diff --git a/makefu/1systems/filepimp.nix b/makefu/1systems/filepimp.nix
index 66ea2ce90..2d008cee6 100644
--- a/makefu/1systems/filepimp.nix
+++ b/makefu/1systems/filepimp.nix
@@ -9,28 +9,35 @@
     [ # Include the results of the hardware scan.
       ../2configs/fs/single-partition-ext4.nix
       ../2configs/tinc-basic-retiolum.nix
+      ../2configs/smart-monitor.nix
     ];
   krebs.build.host = config.krebs.hosts.filepimp;
-
+  services.smartd.devices = [
+    { device = "/dev/sda"; }
+    { device = "/dev/sdb"; }
+    { device = "/dev/sdc"; }
+    { device = "/dev/sdd"; }
+    { device = "/dev/sde"; }
+  ];
   # AMD N54L
   boot = {
-    loader.grub.device = "/dev/sda";
+    loader.grub.device = "/dev/sde";
 
     initrd.availableKernelModules = [
-      "usb_storage"
       "ahci"
-      "xhci_hcd"
-      "ata_piix"
-      "uhci_hcd"
+      "ohci_pci"
       "ehci_pci"
+      "pata_atiixp"
+      "usb_storage"
+      "usbhid"
     ];
 
-    kernelModules = [ ];
+    kernelModules = [ "kvm-amd" ];
     extraModulePackages = [ ];
   };
-
   hardware.enableAllFirmware = true;
   hardware.cpu.amd.updateMicrocode = true;
 
-  networking.firewall.allowPing = true;
+  zramSwap.enable = true;
+  zramSwap.numDevices = 2;
 }
diff --git a/makefu/1systems/omo.nix b/makefu/1systems/omo.nix
index 6ae79398a..e19205a95 100644
--- a/makefu/1systems/omo.nix
+++ b/makefu/1systems/omo.nix
@@ -2,36 +2,95 @@
 # your system.  Help is available in the configuration.nix(5) man page
 # and in the NixOS manual (accessible by running ‘nixos-help’).
 
-{ config, pkgs, ... }:
-
-{
+{ config, pkgs, lib, ... }:
+let
+  byid = dev: "/dev/disk/by-id/" + dev;
+  keyFile = "/dev/disk/by-id/usb-Verbatim_STORE_N_GO_070B3CEE0B223954-0:0";
+  rootDisk = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN";
+  homePartition = byid "ata-INTEL_SSDSA2M080G2GC_CVPO003402PB080BGN-part3";
+  # cryptsetup luksFormat $dev --cipher aes-xts-plain64 -s 512 -h sha512
+  # cryptsetup luksAddKey $dev tmpkey
+  # cryptsetup luksOpen $dev crypt0
+  # mkfs.xfs /dev/mapper/crypt0 -L crypt0
+  cryptDisk0 = byid "ata-ST2000DM001-1CH164_Z240XTT6";
+  cryptDisk1 = byid "ata-TP02000GB_TPW151006050068";
+  cryptDisk2 = byid "ata-WDC_WD20EARS-00MVWB0_WD-WCAZA5548487";
+  # all physical disks
+  allDisks = [ rootDisk cryptDisk0 cryptDisk1 cryptDisk2 ];
+in {
   imports =
-    [ # Include the results of the hardware scan.
+    [
+      # TODO: unlock home partition via ssh
       ../2configs/fs/single-partition-ext4.nix
       ../2configs/tinc-basic-retiolum.nix
+      ../2configs/zsh-user.nix
       ../2configs/exim-retiolum.nix
+      ../2configs/smart-monitor.nix
+      ../2configs/mail-client.nix
+      ../3modules
     ];
   krebs.build.host = config.krebs.hosts.omo;
-
+  services.smartd.devices = builtins.map (x: { device = x; }) allDisks;
+  makefu.snapraid = let
+    toMapper = id: "/media/crypt${builtins.toString id}";
+  in {
+    enable = true;
+    disks = map toMapper [ 0 1 ];
+    parity = toMapper 2;
+  };
   # AMD E350
+  fileSystems = let
+    cryptMount = name:
+      { "/media/${name}" = { device = "/dev/mapper/${name}"; fsType = "xfs"; };};
+  in {
+    "/home" = {
+      device = "/dev/mapper/home";
+      fsType = "ext4";
+    };
+  } // cryptMount "crypt0"
+    // cryptMount "crypt1"
+    // cryptMount "crypt2";
+
+  powerManagement.powerUpCommands = lib.concatStrings (map (disk: ''
+      ${pkgs.hdparm}/sbin/hdparm -S 100 ${disk}
+      ${pkgs.hdparm}/sbin/hdparm -B 127 ${disk}
+      ${pkgs.hdparm}/sbin/hdparm -y ${disk}
+    '') allDisks);
   boot = {
-    loader.grub.device = "/dev/sda";
+    initrd.luks = {
+      devices = let
+        usbkey = name: device: {
+          inherit name device keyFile;
+          keyFileSize = 4096;
+        };
+      in [
+        (usbkey "home" homePartition)
+        (usbkey "crypt0" cryptDisk0)
+        (usbkey "crypt1" cryptDisk1)
+        (usbkey "crypt2" cryptDisk2)
+      ];
+    };
+    loader.grub.device = rootDisk;
 
     initrd.availableKernelModules = [
-      "usb_storage"
       "ahci"
-      "xhci_hcd"
-      "ata_piix"
-      "uhci_hcd"
+      "ohci_pci"
       "ehci_pci"
+      "pata_atiixp"
+      "firewire_ohci"
+      "usb_storage"
+      "usbhid"
     ];
 
-    kernelModules = [ ];
+    kernelModules = [ "kvm-amd" ];
     extraModulePackages = [ ];
   };
 
+  networking.firewall.allowedUDPPorts = [ 655 ];
   hardware.enableAllFirmware = true;
   hardware.cpu.amd.updateMicrocode = true;
 
-  networking.firewall.allowPing = true;
+  #zramSwap.enable = true;
+  zramSwap.numDevices = 2;
+
 }
diff --git a/makefu/1systems/pnp.nix b/makefu/1systems/pnp.nix
index a1b73c0c9..51c124bbe 100644
--- a/makefu/1systems/pnp.nix
+++ b/makefu/1systems/pnp.nix
@@ -1,59 +1,51 @@
-# Edit this configuration file to define what should be installed on
-# your system.  Help is available in the configuration.nix(5) man page
-# and in the NixOS manual (accessible by running ‘nixos-help’).
-
+# Usage:
+#  NIX_PATH=secrets=/home/makefu/secrets/wry:nixpkgs=/var/src/nixpkgs  nix-build -A users.makefu.pnp.config.system.build.vm
+#  result/bin/run-pnp-vm -virtfs local,path=/home/makefu/secrets/pnp,security_model=none,mount_tag=secrets
 { config, pkgs, ... }:
 
 {
   imports =
-    [ # Include the results of the hardware scan.
-      # Base
+    [
       ../2configs/tinc-basic-retiolum.nix
       ../2configs/headless.nix
+      ../../krebs/3modules/Reaktor.nix
 
-      # HW/FS
-
-      # enables virtio kernel modules in initrd
+      # these will be overwritten by qemu-vm.nix but will be used if the system
+      # is directly deployed
       <nixpkgs/nixos/modules/profiles/qemu-guest.nix>
       ../2configs/fs/vm-single-partition.nix
 
-      # Services
-      ../2configs/git/cgit-retiolum.nix
-
-      ## Reaktor
-      ## \/ are only plugins, must enable Reaktor explicitly
-      ../2configs/Reaktor/stockholmLentil.nix
-      ../2configs/Reaktor/simpleExtend.nix
-      ../2configs/Reaktor/random-emoji.nix
-      ../2configs/Reaktor/titlebot.nix
-      ../2configs/Reaktor/shack-correct.nix
-
-      # ../2configs/graphite-standalone.nix
+      # config.system.build.vm
+      <nixpkgs/nixos/modules/virtualisation/qemu-vm.nix>
     ];
-  krebs.urlwatch.verbose = true;
 
-  krebs.Reaktor.enable = true;
-  krebs.Reaktor.debug = true;
-  krebs.Reaktor.nickname = "Reaktor|bot";
-  krebs.Reaktor.extraEnviron = {
-    REAKTOR_CHANNELS = "#krebs,#binaergewitter,#shackspace";
+  virtualisation.graphics = false;
+  # also export secrets, see Usage above
+  fileSystems = pkgs.lib.mkVMOverride {
+    "${builtins.toString <secrets>}" =
+      { device = "secrets";
+        fsType = "9p";
+        options = "trans=virtio,version=9p2000.L,cache=loose";
+        neededForBoot = true;
+      };
+  };
+
+  krebs.Reaktor = {
+    enable = true;
+    debug = true;
+    extraEnviron = {
+      REAKTOR_HOST = "cd.retiolum";
+    };
+    plugins = with pkgs.ReaktorPlugins; [ stockholm-issue nixos-version sed-plugin ];
+    channels = [ "#retiolum" ];
   };
 
   krebs.build.host = config.krebs.hosts.pnp;
 
   nixpkgs.config.packageOverrides = pkgs: { tinc = pkgs.tinc_pre; };
 
-
   networking.firewall.allowedTCPPorts = [
-  # nginx runs on 80
-  80
-  # graphite-web runs on 8080, carbon cache runs on 2003 tcp and udp
-  # 8080 2003
-
-  # smtp
   25
   ];
 
-  # networking.firewall.allowedUDPPorts = [ 2003 ];
-
 }
diff --git a/makefu/2configs/default.nix b/makefu/2configs/default.nix
index a0b49edaf..7593eaff7 100644
--- a/makefu/2configs/default.nix
+++ b/makefu/2configs/default.nix
@@ -23,8 +23,8 @@ with lib;
       source = {
         git.nixpkgs = {
           #url = https://github.com/NixOS/nixpkgs;
-          url = mkDefault https://github.com/makefu/nixpkgs;
-          rev = mkDefault "3fd2c24685f604edc925f73ed56600b8c66236b3"; # nixos-15.09 + cherry-picking
+          url = mkDefault https://github.com/nixos/nixpkgs;
+          rev = mkDefault "93d8671e2c6d1d25f126ed30e5e6f16764330119"; # unstable @ 2015-01-03, tested on filepimp
           target-path = "/var/src/nixpkgs";
         };
 
diff --git a/makefu/2configs/mail-client.nix b/makefu/2configs/mail-client.nix
index a6ae33d2f..bda21e9d0 100644
--- a/makefu/2configs/mail-client.nix
+++ b/makefu/2configs/mail-client.nix
@@ -7,6 +7,8 @@ with lib;
     mutt-kz
     notmuch
     offlineimap
+    imapfilter
+    gnupg
   ];
 
 }
diff --git a/makefu/2configs/smart-monitor.nix b/makefu/2configs/smart-monitor.nix
index 7086f622b..9b0290a9b 100644
--- a/makefu/2configs/smart-monitor.nix
+++ b/makefu/2configs/smart-monitor.nix
@@ -1,5 +1,6 @@
-{ config, ... }:
+{ config, lib, ... }:
 {
+  krebs.exim-retiolum.enable = lib.mkDefault true;
   services.smartd = {
     enable = true;
     notifications = {
@@ -11,7 +12,7 @@
     # short daily, long weekly, check on boot
     defaults.monitored = "-a -o on -s (S/../.././02|L/../../7/04)";
 
-    devices = [{
+    devices = lib.mkDefault [{
       device = "/dev/sda";
     }];
   };
diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix
index a8a1f69d0..218c9138e 100644
--- a/makefu/3modules/default.nix
+++ b/makefu/3modules/default.nix
@@ -2,6 +2,7 @@ _:
 
 {
   imports = [
+    ./snapraid.nix
   ];
 }
 
diff --git a/makefu/3modules/snapraid.nix b/makefu/3modules/snapraid.nix
new file mode 100644
index 000000000..fbdf50219
--- /dev/null
+++ b/makefu/3modules/snapraid.nix
@@ -0,0 +1,125 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  # returns dirname without / , used as disk name
+  dname = dir: replaceChars ["/"] [""] (head (reverseList (splitString "/" dir)));
+  snapraid-conf = ''
+    # Disks
+    ${concatMapStringsSep "\n" (d: "disk ${dname d} ${d}")  cfg.disks}
+    # Parity
+    ${optionalString (cfg.parity != "") "parity ${cfg.parity}/snapraid.parity"}
+
+    # content on Disks
+    ${optionalString cfg.contentOnDisks
+      concatMapStringsSep "\n" (d: "content ${d}/snapraid.content")  cfg.disks}
+
+    # content on Parity
+    ${optionalString (cfg.contentOnParity && cfg.parity != "")
+      "content ${cfg.parity}/snapraid.content"}
+    # Default content file
+    content ${cfg.defaultContentFile}
+
+    # Extra Configuration
+    ${cfg.extraConfig}
+  '';
+  cfg = config.makefu.snapraid;
+
+  out = {
+    options.makefu.snapraid = api;
+    config = mkIf cfg.enable imp;
+  };
+
+  api = {
+    enable = mkEnableOption "snapraid";
+
+    timerConfig = mkOption {
+      type = types.unspecified;
+      description = ''
+        Start snapraid service
+      '';
+      default = {
+        OnCalendar = "daily";
+      };
+    };
+    disks = mkOption {
+      type = with types;listOf str;
+      description = ''
+        Disks to protect. Each disk is a path to the mounted directory of the
+        disk.
+      '';
+    };
+    parity = mkOption {
+      type = types.str;
+      description = ''
+        Folder to store parity file.
+        Set to empty string if you want to configure the parity yourself in
+        extraConfig.
+
+        All extra parity files (2,3,z, etc...) should be configured via
+        extraConfig.
+      '';
+    };
+    contentOnDisks = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Store Content file on each Disk to protect.
+        Set this to false if you do not want this behavior to apply.
+      '';
+    };
+    contentOnParity = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Store Content file on parity Disk.
+        Set this to false if you do not want this behavior to apply.
+      '';
+    };
+    defaultContentFile = mkOption {
+      type = types.str;
+      default = "/var/cache/snapraid.content";
+      description = ''
+        Path to default content file
+        Set to empty string if this content file should be written.
+      '';
+    };
+    extraConfig = mkOption {
+      type = types.string;
+      default = "";
+      description = ''
+        Extra configuration to be appended to the snapraid conf file.
+        You can configure extra Parity files as well as extra content files.
+        See `man snapraid` for additional configuration
+      '';
+    };
+  };
+
+  imp = {
+    environment.systemPackages = [
+      # for scrubbing,fixing
+      pkgs.snapraid
+    ];
+    environment.etc."snapraid.conf".text = snapraid-conf;
+    systemd.timers.snapraid-sync = {
+      description = "snapraid sync timer";
+      wantedBy = [ "timers.target" ];
+      timerConfig = cfg.timerConfig;
+    };
+    systemd.services.snapraid-sync = {
+      description = "Snapraid sync service";
+      after = [ "network.target" "local-fs.target" ];
+
+      serviceConfig = {
+        Type = "simple";
+        ExecStartPre = pkgs.writeScript "Snapraid-sync-init" ''
+          #! /bin/sh
+          ${optionalString (cfg.defaultContentFile != "")
+            "mkdir -p $(dirname ${cfg.defaultContentFile})"}
+        '';
+        ExecStart = "${pkgs.snapraid}/bin/snapraid sync";
+      };
+    };
+  };
+in out
diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix
index 6ffd7fe8a..3275189a5 100644
--- a/shared/2configs/buildbot-standalone.nix
+++ b/shared/2configs/buildbot-standalone.nix
@@ -30,6 +30,7 @@ in {
   # test the master real quick
   sched.append(schedulers.SingleBranchScheduler(
                               change_filter=util.ChangeFilter(branch="master"),
+                              treeStableTimer=10, #only test the latest push
                               name="fast-master-test",
                               builderNames=["fast-tests"]))
         '';
@@ -121,7 +122,7 @@ in {
 
   addShell(s, name="infest-cac-centos7",env=env,
               sigtermTime=60,           # SIGTERM 1 minute before SIGKILL
-              timeout=7200,             # 2h
+              timeout=10800,             # 3h
               command=nixshell + ["infest-cac-centos7"])
 
   bu.append(util.BuilderConfig(name="full-tests",