import stockholm's deployment tools

https://cgit.krebsco.de/stockholm 877b4104370c1ea9698a449e376e2842d7c372fd

9 files changed, 580 insertions, 0 deletions

diff --git a/lib/default.nix b/lib/default.nix
new file mode 100644
index 0000000..7197fe9
--- /dev/null
+++ b/lib/default.nix
@@ -0,0 +1,56 @@
+let {
+
+  body = lib;
+
+  lib = nixpkgs.lib // builtins // {
+
+    evalSource = let
+      eval = source: lib.evalModules {
+        modules = lib.singleton {
+          _file = toString ./.;
+          imports = map (source: { inherit source; }) (lib.toList source);
+          options.source = lib.mkOption {
+            default = {};
+            type = lib.types.attrsOf lib.types.source;
+          };
+        };
+      };
+      sanitize = x: lib.getAttr (lib.typeOf x) {
+        set = lib.mapAttrs
+                (lib.const sanitize)
+                (lib.filterAttrs
+                  (name: value: name != "_module" && value != null) x);
+        string = x;
+      };
+    in
+      # This function's return value can be used as pkgs.populate input.
+      source: sanitize (eval source).config.source;
+
+    getHostName = let
+      # We're parsing /etc/hostname here because reading
+      # /proc/sys/kernel/hostname yields ""
+      y = lib.filter lib.types.label.check (lib.splitString "\n" (lib.readFile /etc/hostname));
+    in
+      if lib.length y != 1 then throw "malformed /etc/hostname" else
+      lib.elemAt y 0;
+
+    mkTarget = s: let
+      default = defVal: val: if val != null then val else defVal;
+      parse = lib.match "(([^@]+)@)?(([^:/]+))?(:([^/]+))?(/.*)?" s;
+      elemAt' = xs: i: if lib.length xs > i then lib.elemAt xs i else null;
+    in {
+      user = default (lib.getEnv "LOGNAME") (elemAt' parse 1);
+      host = default (lib.maybeEnv "HOSTNAME" lib.getHostName) (elemAt' parse 3);
+      port = default "22" /* "ssh"? */ (elemAt' parse 5);
+      path = default "/var/src" /* no default? */ (elemAt' parse 6);
+    };
+
+    test = re: x: lib.isString x && lib.testString re x;
+    testString = re: x: lib.match re x != null;
+
+    types = nixpkgs.lib.types // import ./types { lib = body; };
+  };
+
+  nixpkgs.lib = import <nixpkgs/lib>;
+
+}
diff --git a/lib/types/default.nix b/lib/types/default.nix
new file mode 100644
index 0000000..c4bf517
--- /dev/null
+++ b/lib/types/default.nix
@@ -0,0 +1,8 @@
+{ lib }@args: let {
+
+  body = lib.foldl' (res: path: res // import path args) {} [
+    ./populate.nix
+    ./posix.nix
+  ];
+
+}
diff --git a/lib/types/populate.nix b/lib/types/populate.nix
new file mode 100644
index 0000000..3b13df0
--- /dev/null
+++ b/lib/types/populate.nix
@@ -0,0 +1,93 @@
+{ lib }: rec {
+
+  source = lib.types.submodule ({ config, ... }: {
+    options = {
+      type = let
+        known-types = lib.attrNames source-types;
+        type-candidates = lib.filter (k: config.${k} != null) known-types;
+      in lib.mkOption {
+        default = if lib.length type-candidates == 1
+                    then lib.head type-candidates
+                    else throw "cannot determine type";
+        type = lib.types.enum known-types;
+      };
+      file = lib.mkOption {
+        apply = x:
+          if lib.types.absolute-pathname.check x
+            then { path = x; }
+            else x;
+        default = null;
+        type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.file);
+      };
+      git = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr source-types.git;
+      };
+      pass = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr source-types.pass;
+      };
+      pipe = lib.mkOption {
+        apply = x:
+          if lib.types.absolute-pathname.check x
+            then { command = x; }
+            else x;
+        default = null;
+        type = lib.types.nullOr (lib.types.either lib.types.absolute-pathname source-types.pipe);
+      };
+      symlink = lib.mkOption {
+        apply = x:
+          if lib.types.pathname.check x
+            then { target = x; }
+            else x;
+        default = null;
+        type = lib.types.nullOr (lib.types.either lib.types.pathname source-types.symlink);
+      };
+    };
+  });
+
+  source-types = {
+    file = lib.types.submodule {
+      options = {
+        path = lib.mkOption {
+          type = lib.types.absolute-pathname;
+        };
+      };
+    };
+    git = lib.types.submodule {
+      options = {
+        ref = lib.mkOption {
+          type = lib.types.str; # TODO lib.types.git.ref
+        };
+        url = lib.mkOption {
+          type = lib.types.str; # TODO lib.types.git.url
+        };
+      };
+    };
+    pass = lib.types.submodule {
+      options = {
+        dir = lib.mkOption {
+          type = lib.types.absolute-pathname;
+        };
+        name = lib.mkOption {
+          type = lib.types.pathname; # TODO relative-pathname
+        };
+      };
+    };
+    pipe = lib.types.submodule {
+      options = {
+        command = lib.mkOption {
+          type = lib.types.absolute-pathname;
+        };
+      };
+    };
+    symlink = lib.types.submodule {
+      options = {
+        target = lib.mkOption {
+          type = lib.types.pathname; # TODO relative-pathname
+        };
+      };
+    };
+  };
+
+}
diff --git a/lib/types/posix.nix b/lib/types/posix.nix
new file mode 100644
index 0000000..e8f464e
--- /dev/null
+++ b/lib/types/posix.nix
@@ -0,0 +1,54 @@
+{ lib }: rec {
+
+  # RFC952, B. Lexical grammar, <hname>
+  hostname = lib.mkOptionType {
+    name = "hostname";
+    check = x: lib.isString x && lib.all label.check (lib.splitString "." x);
+    merge = lib.mergeOneOption;
+  };
+
+  # RFC952, B. Lexical grammar, <name>
+  # RFC1123, 2.1  Host Names and Numbers
+  label = lib.mkOptionType {
+    name = "label";
+    # TODO case-insensitive labels
+    check = lib.test "[0-9A-Za-z]([0-9A-Za-z-]*[0-9A-Za-z])?";
+    merge = lib.mergeOneOption;
+  };
+
+  # POSIX.1‐2013, 3.278 Portable Filename Character Set
+  filename = lib.mkOptionType {
+    name = "POSIX filename";
+    check = lib.test "([0-9A-Za-z._])[0-9A-Za-z._-]*";
+    merge = lib.mergeOneOption;
+  };
+
+  # POSIX.1‐2013, 3.2 Absolute Pathname
+  absolute-pathname = lib.mkOptionType {
+    name = "POSIX absolute pathname";
+    check = x: lib.isString x && lib.substring 0 1 x == "/" && pathname.check x;
+    merge = lib.mergeOneOption;
+  };
+
+  # POSIX.1‐2013, 3.267 Pathname
+  pathname = lib.mkOptionType {
+    name = "POSIX pathname";
+    check = x:
+      let
+        # The filter is used to normalize paths, i.e. to remove duplicated and
+        # trailing slashes.  It also removes leading slashes, thus we have to
+        # check for "/" explicitly below.
+        xs = lib.filter (s: lib.stringLength s > 0) (lib.splitString "/" x);
+      in
+        lib.isString x && (x == "/" || (lib.length xs > 0 && lib.all filename.check xs));
+    merge = lib.mergeOneOption;
+  };
+
+  # POSIX.1-2013, 3.431 User Name
+  username = lib.mkOptionType {
+    name = "POSIX username";
+    check = filename.check;
+    merge = lib.mergeOneOption;
+  };
+
+}
diff --git a/pkgs/default.nix b/pkgs/default.nix
new file mode 100644
index 0000000..639ed13
--- /dev/null
+++ b/pkgs/default.nix
@@ -0,0 +1,7 @@
+{ overlays ? [], ... }@args:
+
+import <nixpkgs> (args // {
+  overlays = overlays ++ [
+    (import ./overlay.nix)
+  ];
+})
diff --git a/pkgs/kops/default.nix b/pkgs/kops/default.nix
new file mode 100644
index 0000000..fc52327
--- /dev/null
+++ b/pkgs/kops/default.nix
@@ -0,0 +1,48 @@
+let
+  lib = import ../../lib // {
+    isLocalTarget = let
+      origin = lib.mkTarget "";
+    in target:
+      target.host == origin.host &&
+      target.user == origin.user;
+  };
+in
+
+{ nix, openssh, populate, writeDash, writeJSON }: {
+
+  writeDeploy = name: { source, target }: let
+    target' = lib.mkTarget target;
+  in
+    writeDash name ''
+      set -efu
+
+      ${populate}/bin/populate \
+          ${target'.user}@${target'.host}:${target'.port}${target'.path} \
+        < ${writeJSON "${name}-source.json" source}
+
+      ${openssh}/bin/ssh \
+          ${target'.user}@${target'.host} -p ${target'.port} \
+          nixos-rebuild switch -I ${target'.path}
+    '';
+
+  writeTest = name: { source, target }: let
+    target' = lib.mkTarget target;
+  in
+    assert lib.isLocalTarget target';
+    writeDash name ''
+      set -efu
+
+      ${populate}/bin/populate --force \
+          ${target'.path} \
+        < ${writeJSON "${name}-source.json" source}
+
+      ${nix}/bin/nix-build \
+          -A config.system.build.toplevel \
+          -I ${target'.path} \
+          --arg modules '[<nixos-config>]' \
+          --no-out-link \
+          --show-trace \
+          '<nixpkgs/nixos/lib/eval-config.nix>'
+    '';
+
+}
diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix
new file mode 100644
index 0000000..89024bd
--- /dev/null
+++ b/pkgs/overlay.nix
@@ -0,0 +1,18 @@
+let
+  lib = import ../lib;
+in
+
+self: super: {
+  kops = self.callPackage ./kops {};
+  populate = self.callPackage ./populate {};
+  writeDash = name: text: self.writeScript name ''
+    #! ${self.dash}/bin/dash
+    ${text}
+  '';
+  writeJSON = name: value: self.runCommand name {
+    json = lib.toJSON value;
+    passAsFile = [ "json" ];
+  } /* sh */ ''
+    ${self.jq}/bin/jq . "$jsonPath" > "$out"
+  '';
+}
diff --git a/pkgs/populate/default.nix b/pkgs/populate/default.nix
new file mode 100644
index 0000000..acb5a5f
--- /dev/null
+++ b/pkgs/populate/default.nix
@@ -0,0 +1,20 @@
+{ coreutils, findutils, git, gnused, jq, openssh, pass, rsync, runCommand, stdenv }:
+
+let
+  PATH = stdenv.lib.makeBinPath [
+    coreutils
+    findutils
+    git
+    gnused
+    jq
+    openssh
+    pass
+    rsync
+  ];
+in
+
+runCommand "populate-2.2.0" {} ''
+  mkdir -p $out/bin
+  cp ${./populate.sh} $out/bin/populate
+  sed -i '1s,.*,&\nPATH=${PATH},' $out/bin/populate
+''
diff --git a/pkgs/populate/populate.sh b/pkgs/populate/populate.sh
new file mode 100755
index 0000000..670d25b
--- /dev/null
+++ b/pkgs/populate/populate.sh
@@ -0,0 +1,276 @@
+#! /bin/sh
+set -efu
+
+main() {(
+  self=$(readlink -f "$0")
+  basename=${0##*/}
+
+  debug=false
+  force=false
+  origin_host=${HOSTNAME-cat /proc/sys/kernel/hostname}
+  origin_user=$LOGNAME
+  target_spec=
+
+
+  abort=false
+
+  error() {
+    echo "$basename: error: $1" >&2
+    abort=true
+  }
+
+  for arg; do
+    case $arg in
+      --force)
+        force=true
+        ;;
+      -*)
+        error "bad argument: $arg"
+        ;;
+      *)
+        if test -n "$target_spec"; then
+          error "bad argument: $arg"
+        else
+          target_spec=$arg
+        fi
+        ;;
+    esac
+  done
+
+  if test -z "$target_spec"; then
+    error 'no target specified'
+  fi
+
+  if test "$abort" = true; then
+    exit 11
+  fi
+
+  target=$(
+    export origin_host
+    export origin_user
+    echo "$target_spec" | jq -R '
+      def default(value; f): if . == null then value else f end;
+      def default(value): default(value; .);
+
+      match("^(?:([^@]+)@)?(?:([^:/]+))?(?::([^/]+))?(/.*)?")
+      | {
+        user: .captures[0].string | default(env.origin_user),
+        host: .captures[1].string | default(env.origin_host),
+        port: .captures[2].string | default(22;
+          if test("^[0-9]+$") then fromjson else
+            error(@json "bad target port: \(.)")
+          end),
+        path: .captures[3].string | default("/var/src"),
+      }
+    '
+  )
+
+  echo $target | jq . >&2
+
+  target_host=$(echo $target | jq -r .host)
+  target_path=$(echo $target | jq -r .path)
+  target_port=$(echo $target | jq -r .port)
+  target_user=$(echo $target | jq -r .user)
+
+  if test "$force" = true; then
+    force_target
+  else
+    check_target
+  fi
+
+  jq -c 'to_entries | group_by(.value.type) | flatten[]' |
+  while read -r source; do
+    key=$(echo "$source" | jq -r .key)
+    type=$(echo "$source" | jq -r .value.type)
+    conf=$(echo "$source" | jq -r .value.${type})
+
+    printf '\e[1;33m%s\e[m\n' "populate_$type $key $conf" >&2
+
+    populate_"$type" "$key" "$conf"
+  done
+)}
+
+# Safeguard to prevent clobbering of misspelled targets.
+# This function has to be called first.
+check_target() {
+  {
+    echo target_host=$(quote "$target_host")
+    echo target_path=$(quote "$target_path")
+    echo 'sentinel_file=$target_path/.populate'
+    echo 'if ! test -f "$sentinel_file"; then'
+    echo '  echo "error: missing sentinel file: $target_host:$sentinel_file" >&2'
+    echo '  exit 1'
+    echo 'fi'
+  } \
+    |
+  target_shell
+}
+
+force_target() {
+  {
+    echo target_path=$(quote "$target_path")
+    echo 'sentinel_file=$target_path/.populate'
+    echo 'mkdir -vp "$target_path"'
+    echo 'touch "$sentinel_file"'
+  } \
+    |
+  target_shell
+}
+
+is_local_target() {
+  test "$target_host" = "$origin_host" &&
+  test "$target_user" = "$origin_user"
+}
+
+populate_file() {(
+  file_name=$1
+  file_path=$(echo "$2" | jq -r .path)
+
+  if is_local_target; then
+    file_target=$target_path/$file_name
+  else
+    file_target=$target_user@$target_host:$target_path/$file_name
+  fi
+
+  rsync \
+      -vFrlptD \
+      --delete-excluded \
+      "$file_path"/ \
+      -e "ssh -o ControlPersist=no -p $target_port" \
+      "$file_target"
+)}
+
+populate_git() {(
+  git_name=$1
+  git_url=$(echo "$2" | jq -r .url)
+  git_ref=$(echo "$2" | jq -r .ref)
+
+  git_work_tree=$target_path/$git_name
+
+  {
+    echo set -efu
+
+    echo git_url=$(quote "$git_url")
+    echo git_ref=$(quote "$git_ref")
+
+    echo git_work_tree=$(quote "$git_work_tree")
+
+    echo 'if ! test -e "$git_work_tree"; then'
+    echo '  git clone "$git_url" "$git_work_tree"'
+    echo 'fi'
+
+    echo 'cd $git_work_tree'
+
+    echo 'if ! url=$(git config remote.origin.url); then'
+    echo '  git remote add origin "$git_url"'
+    echo 'elif test "$url" != "$git_url"; then'
+    echo '  git remote set-url origin "$git_url"'
+    echo 'fi'
+
+    # TODO resolve git_ref to commit hash
+    echo 'hash=$git_ref'
+
+    echo 'if ! test "$(git log --format=%H -1)" = "$hash"; then'
+    echo '  if ! git log -1 "$hash" >/dev/null 2>&1; then'
+    echo '    git fetch origin'
+    echo '  fi'
+    echo '  git checkout "$hash" -- "$git_work_tree"'
+    echo '  git -c advice.detachedHead=false checkout -f "$hash"'
+    echo 'fi'
+
+    echo 'git clean -dfx'
+
+  } \
+    |
+  target_shell
+)}
+
+populate_pass() {(
+  pass_target_name=$1
+  pass_dir=$(echo "$2" | jq -r .dir)
+  pass_name_root=$(echo "$2" | jq -r .name)
+
+  if is_local_target; then
+    pass_target=$target_path/$pass_target_name
+  else
+    pass_target=$target_user@$target_host:$target_path/$pass_target_name
+  fi
+
+  umask 0077
+
+  tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX)
+  trap cleanup EXIT
+  cleanup() {
+    rm -fR "$tmp_dir"
+  }
+
+  pass_prefix=$pass_dir/$pass_name_root/
+
+  find "$pass_prefix" -type f |
+  while read -r pass_gpg_file_path; do
+
+    rel_name=${pass_gpg_file_path:${#pass_prefix}}
+    rel_name=${rel_name%.gpg}
+
+    pass_name=$pass_name_root/$rel_name
+    tmp_path=$tmp_dir/$rel_name
+
+    mkdir -p "$(dirname "$tmp_path")"
+    PASSWORD_STORE_DIR=$pass_dir pass show "$pass_name" > "$tmp_path"
+  done
+
+  rsync \
+      --checksum \
+      -vFrlptD \
+      --delete-excluded \
+      "$tmp_dir"/ \
+      -e "ssh -o ControlPersist=no -p $target_port" \
+      "$pass_target"
+)}
+
+populate_pipe() {(
+  pipe_target_name=$1
+  pipe_command=$(echo "$2" | jq -r .command)
+
+  result_path=$target_path/$pipe_target_name
+
+  "$pipe_command" | target_shell -c "cat > $(quote "$result_path")"
+)}
+
+populate_symlink() {(
+  symlink_name=$1
+  symlink_target=$(echo "$2" | jq -r .target)
+  link_name=$target_path/$symlink_name
+
+  {
+    # TODO rm -fR instead of ln -f?
+    echo ln -fns $(quote "$symlink_target" "$link_name")
+  } \
+    |
+  target_shell
+)}
+
+quote() {
+  printf %s "$1" | sed 's/./\\&/g'
+  while test $# -gt 1; do
+    printf ' '
+    shift
+    printf %s "$1" | sed 's/./\\&/g'
+  done
+  echo
+}
+
+target_shell() {
+  if is_local_target; then
+    /bin/sh "$@"
+  else
+    ssh "$target_host" \
+        -l "$target_user" \
+        -o ControlPersist=no \
+        -p "$target_port" \
+        -T \
+        /bin/sh "$@"
+  fi
+}
+
+main "$@"