From 36670f3e1cccad579b994a29320eeb8e287803b7 Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 21 May 2015 01:56:08 +0200 Subject: sh: functions -> bin/ --- bin/_cac_curl_api_v1 | 10 ++ bin/_cac_exec | 8 ++ bin/_cac_get_api_v1 | 3 + bin/_cac_post_api_v1 | 3 + bin/bre-escape | 5 + bin/bre-invert-word | 15 +++ bin/cac-cloudpro-build | 5 + bin/cac-cloudpro-delete | 3 + bin/cac-cloudpro-resources | 3 + bin/cac-console | 3 + bin/cac-get-server-by | 15 +++ bin/cac-listservers | 12 +++ bin/cac-listtasks | 3 + bin/cac-listtemplates | 4 + bin/cac-powerop | 3 + bin/cac-rdns | 3 + bin/cac-renameserver | 3 + bin/cac-runmode | 3 + bin/cac-ssh | 17 +++ bin/cacnixos-networking | 28 +++++ bin/filter-secrets | 6 ++ bin/import-statements | 10 ++ bin/infest-CentOS-7-64bit | 150 ++++++++++++++++++++++++++ bin/infest-cac | 21 ++++ bin/json-assert-type | 18 ++++ bin/list-hosts | 7 ++ bin/list-module-imports | 20 ++++ bin/ls-bre | 12 +++ bin/make-parent-dirs | 10 ++ bin/make-relative-to | 6 ++ bin/make-rsync-filter | 33 ++++++ bin/make-rsync-whitelist | 15 +++ bin/netmask-to-prefix | 12 +++ bin/nixpkgs-rev | 13 +++ bin/nixpkgs-url | 13 +++ bin/quoted-strings | 15 +++ bin/slash-path-relpath | 8 ++ bin/ssh-deploy | 26 +++++ bin/ssh-fetch-git | 35 ++++++ bin/undot-paths | 14 +++ bin/urlencode | 35 ++++++ deploy | 5 +- infest | 188 ++------------------------------ lib/cac.sh | 105 ------------------ lib/cacnixos.sh | 28 ----- lib/net.sh | 9 -- lib/prelude.sh | 261 --------------------------------------------- lib/url.sh | 35 ------ 48 files changed, 639 insertions(+), 620 deletions(-) create mode 100755 bin/_cac_curl_api_v1 create mode 100755 bin/_cac_exec create mode 100755 bin/_cac_get_api_v1 create mode 100755 bin/_cac_post_api_v1 create mode 100755 bin/bre-escape create mode 100755 bin/bre-invert-word create mode 100755 bin/cac-cloudpro-build create mode 100755 bin/cac-cloudpro-delete create mode 100755 bin/cac-cloudpro-resources create mode 100755 bin/cac-console create mode 100755 bin/cac-get-server-by create mode 100755 bin/cac-listservers create mode 100755 bin/cac-listtasks create mode 100755 bin/cac-listtemplates create mode 100755 bin/cac-powerop create mode 100755 bin/cac-rdns create mode 100755 bin/cac-renameserver create mode 100755 bin/cac-runmode create mode 100755 bin/cac-ssh create mode 100755 bin/cacnixos-networking create mode 100755 bin/filter-secrets create mode 100755 bin/import-statements create mode 100755 bin/infest-CentOS-7-64bit create mode 100755 bin/infest-cac create mode 100755 bin/json-assert-type create mode 100755 bin/list-hosts create mode 100755 bin/list-module-imports create mode 100755 bin/ls-bre create mode 100755 bin/make-parent-dirs create mode 100755 bin/make-relative-to create mode 100755 bin/make-rsync-filter create mode 100755 bin/make-rsync-whitelist create mode 100755 bin/netmask-to-prefix create mode 100755 bin/nixpkgs-rev create mode 100755 bin/nixpkgs-url create mode 100755 bin/quoted-strings create mode 100755 bin/slash-path-relpath create mode 100755 bin/ssh-deploy create mode 100755 bin/ssh-fetch-git create mode 100755 bin/undot-paths create mode 100755 bin/urlencode delete mode 100644 lib/cac.sh delete mode 100644 lib/cacnixos.sh delete mode 100644 lib/net.sh delete mode 100644 lib/prelude.sh delete mode 100644 lib/url.sh diff --git a/bin/_cac_curl_api_v1 b/bin/_cac_curl_api_v1 new file mode 100755 index 00000000..65acebd9 --- /dev/null +++ b/bin/_cac_curl_api_v1 @@ -0,0 +1,10 @@ +#! /bin/sh +set -euf + +exec _cac_exec curl -fsS "$1" "https://panel.cloudatcost.com/api/v1/$2.php" $( + shift 2 + set -- "$@" login="$cac_login" key="$cac_key" + for arg; do + echo -d $(printf '%s' "$arg" | urlencode) + done +) diff --git a/bin/_cac_exec b/bin/_cac_exec new file mode 100755 index 00000000..c932454e --- /dev/null +++ b/bin/_cac_exec @@ -0,0 +1,8 @@ +#! /bin/sh +set -euf + +if test -z "${cac_via-}"; then + exec "$@" +else + exec ssh -q "$cac_via" -t "$@" +fi diff --git a/bin/_cac_get_api_v1 b/bin/_cac_get_api_v1 new file mode 100755 index 00000000..67aac856 --- /dev/null +++ b/bin/_cac_get_api_v1 @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_curl_api_v1 -G "$@" diff --git a/bin/_cac_post_api_v1 b/bin/_cac_post_api_v1 new file mode 100755 index 00000000..b946ed9f --- /dev/null +++ b/bin/_cac_post_api_v1 @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_curl_api_v1 -XPOST "$@" diff --git a/bin/bre-escape b/bin/bre-escape new file mode 100755 index 00000000..ae961b0e --- /dev/null +++ b/bin/bre-escape @@ -0,0 +1,5 @@ +#! /bin/sh +# bre-escape : lines string |> lines bre-escaped-string +set -euf + +sed 's:[\.\[\\\*\^\$]:\\&:g' diff --git a/bin/bre-invert-word b/bin/bre-invert-word new file mode 100755 index 00000000..677ba2e9 --- /dev/null +++ b/bin/bre-invert-word @@ -0,0 +1,15 @@ +#! /bin/sh +# bre-invert-word : string -> BRE +set -euf + +# TODO escape chars in the resulting BRE. +awk -v input="$1" ' + BEGIN { + split(input,s,"") + for (i in s) { + c=s[i] + printf "\\|%s[^%s]", y, c + y = y c + } + } +' diff --git a/bin/cac-cloudpro-build b/bin/cac-cloudpro-build new file mode 100755 index 00000000..782fa0d7 --- /dev/null +++ b/bin/cac-cloudpro-build @@ -0,0 +1,5 @@ +#! /bin/sh +set -euf + +# default os=26 is CentOS-7-64bit +exec _cac_post_api_v1 cloudpro/build cpu="$1" ram="$2" storage="$3" os="${4-26}" diff --git a/bin/cac-cloudpro-delete b/bin/cac-cloudpro-delete new file mode 100755 index 00000000..ee1dbbc7 --- /dev/null +++ b/bin/cac-cloudpro-delete @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 cloudpro/delete sid="$1" diff --git a/bin/cac-cloudpro-resources b/bin/cac-cloudpro-resources new file mode 100755 index 00000000..9ec5872e --- /dev/null +++ b/bin/cac-cloudpro-resources @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_get_api_v1 cloudpro/resources diff --git a/bin/cac-console b/bin/cac-console new file mode 100755 index 00000000..ed1cbd5f --- /dev/null +++ b/bin/cac-console @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 console sid="$1" diff --git a/bin/cac-get-server-by b/bin/cac-get-server-by new file mode 100755 index 00000000..b4606251 --- /dev/null +++ b/bin/cac-get-server-by @@ -0,0 +1,15 @@ +#! /bin/sh +set -euf + +cac-listservers \ + | jq \ + --arg k "$1" \ + --arg v "$2" \ + ' + map(select(.[$k]==$v)) | + if (. | length) == 1 then + .[0] + else + . + end + ' diff --git a/bin/cac-listservers b/bin/cac-listservers new file mode 100755 index 00000000..1e815d2a --- /dev/null +++ b/bin/cac-listservers @@ -0,0 +1,12 @@ +#! /bin/sh +set -euf + +listservers=$(_cac_get_api_v1 listservers) +status=$(echo "$listservers" | jq -r .status) + +if [ "$status" = ok ]; then + echo "$listservers" | jq -r .data +else + echo "$0: bad listservers status: $status" >&2 + exit 1 +fi diff --git a/bin/cac-listtasks b/bin/cac-listtasks new file mode 100755 index 00000000..14be3948 --- /dev/null +++ b/bin/cac-listtasks @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_get_api_v1 listtasks diff --git a/bin/cac-listtemplates b/bin/cac-listtemplates new file mode 100755 index 00000000..c4414e01 --- /dev/null +++ b/bin/cac-listtemplates @@ -0,0 +1,4 @@ +#! /bin/sh +set -euf + +exec _cac_get_api_v1 listtemplates diff --git a/bin/cac-powerop b/bin/cac-powerop new file mode 100755 index 00000000..c897835f --- /dev/null +++ b/bin/cac-powerop @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 powerop sid="$1" action="$2" diff --git a/bin/cac-rdns b/bin/cac-rdns new file mode 100755 index 00000000..c2d9ecda --- /dev/null +++ b/bin/cac-rdns @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 rdns sid="$1" hostname="$2" diff --git a/bin/cac-renameserver b/bin/cac-renameserver new file mode 100755 index 00000000..f0eff9b3 --- /dev/null +++ b/bin/cac-renameserver @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 renameserver sid="$1" name="$2" diff --git a/bin/cac-runmode b/bin/cac-runmode new file mode 100755 index 00000000..200b9fb7 --- /dev/null +++ b/bin/cac-runmode @@ -0,0 +1,3 @@ +#! /bin/sh +set -euf +exec _cac_post_api_v1 rdns sid="$1" mode="$2" diff --git a/bin/cac-ssh b/bin/cac-ssh new file mode 100755 index 00000000..a0ec5dcf --- /dev/null +++ b/bin/cac-ssh @@ -0,0 +1,17 @@ +#! /bin/sh +set -euf + +server=$1 +shift + +address=$(echo $server | jq -r .ip) +target=root@$address + +SSHPASS=$(echo $server | jq -r .rootpass) +export SSHPASS + +exec sshpass -e ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "$target" \ + "$@" diff --git a/bin/cacnixos-networking b/bin/cacnixos-networking new file mode 100755 index 00000000..4b246ebf --- /dev/null +++ b/bin/cacnixos-networking @@ -0,0 +1,28 @@ +#! /bin/sh +# cacnixos-networking : cac-server x hostname -> nixos-module +# TODO use label for hostname +set -euf + +server=$1 +hostname=$2 + +address=$(echo $server | jq -r .ip) +gateway=$(echo $server | jq -r .gateway) +nameserver=8.8.8.8 +netmask=$(echo $server | jq -r .netmask) +prefix=$(netmask-to-prefix $netmask) + +printf '{...}:\n' +printf '{\n' +printf ' networking.hostName = "%s";\n' $hostname +printf ' networking.interfaces.enp2s1.ip4 = [\n' +printf ' {\n' +printf ' address = "%s";\n' $address +printf ' prefixLength = %d;\n' $prefix +printf ' }\n' +printf ' ];\n' +printf ' networking.defaultGateway = "%s";\n' $gateway +printf ' networking.nameservers = [\n' +printf ' "%s"\n' $nameserver +printf ' ];\n' +printf '}\n' diff --git a/bin/filter-secrets b/bin/filter-secrets new file mode 100755 index 00000000..6fcce73c --- /dev/null +++ b/bin/filter-secrets @@ -0,0 +1,6 @@ +#! /bin/sh +# filter_secrets : lines string |> lines secrets-file-candidate +set -euf + +# Notice how false positives are possible. +sed -n 's:^\(.*/\)\?\(secrets/.*\):'"${PWD//:/\\:}"'/\2:p' diff --git a/bin/import-statements b/bin/import-statements new file mode 100755 index 00000000..12c88797 --- /dev/null +++ b/bin/import-statements @@ -0,0 +1,10 @@ +#! /bin/sh +# import-statements : lines (path ":" string) |> lines (path ":" relpath) +set -euf +sed -n ' + s@^\([^:]\+:\)\('"$(bre-invert-word import)"'\)*\ modules/$hostname/networking.nix + +echo '( + set -xeuf + type bzip2 || yum install -y bzip2 + type rsync || yum install -y rsync +)' \ + | sshpass -e ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "root@$address" \ + /bin/sh + +make-rsync-filter "$main" \ + | rsync -f '. -' -zvrlptD --delete-excluded ./ "$target":/etc/nixos/ + +# +# +# +echo '( + set -xeuf + groupadd -g 30000 nixbld || : + for i in `seq 1 10`; do + useradd -c "foolsgarden Nix build user $i" \ + -d /var/empty \ + -s /sbin/nologin \ + -g 30000 \ + -G 30000 \ + -l -u $(expr 30000 + $i) \ + nixbld$i || : + rm -f /var/spool/mail/nixbld$i + done + + #curl https://nixos.org/nix/install | sh + nix_tar=$nix_basename.tar.bz2 + if ! echo $nix_sha256 $nix_tar | sha256sum -c; then + curl -O -C - $nix_url || : + if ! echo $nix_sha256 $nix_tar | sha256sum -c; then + curl -O $nix_url || : + if ! echo $nix_sha256 $nix_tar | sha256sum -c; then + echo $0: cannot download $nix_url >&2 + exit 5 + fi + fi + fi + + if ! test -d $nix_basename; then + tar jxf $nix_basename.tar.bz2 + fi + + nix_find=$nix_basename.find.txt + if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then + find $nix_basename | sort > $nix_find + if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then + echo $0: cannot unpack $nix_basename.tar.bz2 >&2 + # TODO we could retry + exit 6 + fi + fi + + mkdir -p bin + PATH=$HOME/bin:$PATH + export PATH + + # generate fake sudo because + # sudo: sorry, you must have a tty to run sudo + { + echo "#! /bin/sh" + echo "exec env \"\$@\"" + } > bin/sudo + chmod +x bin/sudo + + ./$nix_basename/install + + . /root/.nix-profile/etc/profile.d/nix.sh + + nixpkgs_expr="import { system = builtins.currentSystem; }" + nixpkgs_path=$( + find /nix/store -mindepth 1 -maxdepth 1 -name *-nixpkgs-* -type d + ) + + for i in nixos-generate-config nixos-install; do + nix-env \ + --arg config "{ nix.package = ($nixpkgs_expr).nix; }" \ + --arg pkgs "$nixpkgs_expr" \ + --arg modulesPath "throw \"no modulesPath\"" \ + -f $nixpkgs_path/nixpkgs/nixos/modules/installer/tools/tools.nix \ + -iA config.system.build.$i + done + + # TODO following fail when aborted in-between + if ! test -d /int; then + mkdir -p /int + mount --bind /int /mnt + fi + if ! test -d /mnt/boot; then + mkdir -p /mnt/boot + mount /dev/sda1 /mnt/boot + fi + + mkdir -p /mnt/etc/nixos + rsync -zvrlptD --delete-excluded /etc/nixos/ /mnt/etc/nixos/ + + mkdir -m 0444 -p /mnt/var/empty + + ln -s $main /mnt/etc/nixos/configuration.nix + nixos-install \ + -I secrets=/etc/nixos/secrets + + find / \ + 1> /root/pre-rsync-find.out \ + 2> /root/pre-rsync-find.err + + rsync -va --force /int/ / + + # find / -type f -mtime +1 -exec rm -v {} \; 2>&1 > rm.log + # ^ too aggressive, kills journal which is bad + # shutdown -r now + # nix-channel --add https://nixos.org/channels/nixos-unstable nixos + # nix-channel --remove nixpkgs + # nix-channel --update + +)' \ + | sshpass -e ssh \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + "root@$address" \ + -T /usr/bin/env \ + nix_url="$nix_url" \ + nix_basename="$(basename $nix_url .tar.bz2)" \ + nix_sha256="$nix_sha256" \ + nix_find_sha1sum="$nix_find_sha1sum" \ + main="$main" \ + /bin/sh diff --git a/bin/infest-cac b/bin/infest-cac new file mode 100755 index 00000000..d7d7bb96 --- /dev/null +++ b/bin/infest-cac @@ -0,0 +1,21 @@ +#! /bin/sh +set -euf + +server=$(cac-get-server-by servername "$1") +hostname=$2 + +serverstatus=$(echo $server | jq -r .status) +case $serverstatus in + 'Powered On') : ;; + *) + echo $0: bad server status: $serverstatus >&2 + exit 2 +esac + +template=$(echo $server | jq -r .template) +case $template in + 'CentOS-7-64bit') infest-"$template" "$server" "$hostname";; + *) + echo $0: bad template: $template >&2 + exit 3 +esac diff --git a/bin/json-assert-type b/bin/json-assert-type new file mode 100755 index 00000000..29cadad6 --- /dev/null +++ b/bin/json-assert-type @@ -0,0 +1,18 @@ +#! /bin/sh +set -euf + +formal_type=$1 + +actual_value=$2 +actual_type=$(echo $actual_value | jq -r type) + +if [ "$actual_type" != "$formal_type" ]; then + backtrace + printf 'error: expected %s, got %s\n' \ + "$formal_type" \ + "$actual_type" \ + >&2 + exit 1 +fi + +echo "$actual_value" diff --git a/bin/list-hosts b/bin/list-hosts new file mode 100755 index 00000000..e25a8ac4 --- /dev/null +++ b/bin/list-hosts @@ -0,0 +1,7 @@ +#! /bin/sh +# list-hosts : lines tinc-host-file +set -euf + +# Precondition: $PWD/hosts is the correct repository :) +git -C hosts ls-tree --name-only HEAD \ + | awk '{print ENVIRON["PWD"]"/hosts/"$$0}' diff --git a/bin/list-module-imports b/bin/list-module-imports new file mode 100755 index 00000000..39d11bf3 --- /dev/null +++ b/bin/list-module-imports @@ -0,0 +1,20 @@ +#! /bin/sh +# list-module-imports : nix-file -> lines nix-file +set -euf + +if echo "$1" | grep -q ^/; then + : +else + set -- "./$1" +fi + +imports=$(nix-instantiate \ + -I secrets=secrets \ + --strict \ + --json \ + --eval \ + -E \ + "with builtins; with import ./lib/modules.nix; map toString (list-imports $1)") + +echo "$imports" \ + | jq -r .[] diff --git a/bin/ls-bre b/bin/ls-bre new file mode 100755 index 00000000..ae978895 --- /dev/null +++ b/bin/ls-bre @@ -0,0 +1,12 @@ +#! /bin/sh +# ls-bre : directory -> BRE +# Create a BRE from the files in a directory. +set -euf + +ls "$1" \ + | tr \\n / \ + | sed ' + s:[\.\[\\\*\^\$]:\\&:g + s:/$:: + s:/:\\|:g + ' diff --git a/bin/make-parent-dirs b/bin/make-parent-dirs new file mode 100755 index 00000000..f4717b24 --- /dev/null +++ b/bin/make-parent-dirs @@ -0,0 +1,10 @@ +#! /bin/sh +# make-parent-dirs : lines path |> lines directory +# List all parent directories of a path. +set -euf + +set -- "$(sed -n 's|/[^/]*$||p' | grep . | sort | uniq)" +if echo "$1" | grep -q .; then + echo "$1" + echo "$1" | make-parent-dirs +fi diff --git a/bin/make-relative-to b/bin/make-relative-to new file mode 100755 index 00000000..9d947e17 --- /dev/null +++ b/bin/make-relative-to @@ -0,0 +1,6 @@ +#! /bin/sh +# make-relative-to : lines path |> directory -> lines path +# Non-matching paths won't get altered. +set -euf + +sed "s:^$(echo "$1/" | bre-escape | sed 's/:/\\:/g')::" diff --git a/bin/make-rsync-filter b/bin/make-rsync-filter new file mode 100755 index 00000000..26e070ad --- /dev/null +++ b/bin/make-rsync-filter @@ -0,0 +1,33 @@ +#! /bin/sh +# make-rsync-filter : nixos-config -> rsync-filter +set -euf + +main=$1 + +hosts=$(list-hosts) +module_imports=$(list-module-imports "$main") +other_imports=$( + echo "$module_imports" \ + | xargs grep -H . \ + | import-statements \ + | slash-path-relpath \ + | undot-paths \ + | sort \ + | uniq \ + | sed '/\.nix$/!s:$:/default.nix:' \ + ) +secrets=$(echo "$module_imports" | xargs cat | quoted-strings | filter-secrets) + +# TODO collect all other paths from *_imports + +abs_deps=$( + echo "$hosts" + echo "$module_imports" + echo "$other_imports" + echo "$secrets" +) + +rel_deps=$(echo "$abs_deps" | make-relative-to "$PWD") +filter=$(echo "$rel_deps" | make-rsync-whitelist) + +echo "$filter" diff --git a/bin/make-rsync-whitelist b/bin/make-rsync-whitelist new file mode 100755 index 00000000..a1b09c80 --- /dev/null +++ b/bin/make-rsync-whitelist @@ -0,0 +1,15 @@ +#! /bin/sh +# make-rsync-whitelist : lines relpath |> liens rsync-filter +set -euf + +set -- "$(cat)" + +# include all files in stdin and their directories +{ + echo "$1" + echo "$1" | make-parent-dirs | sort | uniq +} \ + | sed 's|^|+ /|' + +# exclude everything else +echo '- *' diff --git a/bin/netmask-to-prefix b/bin/netmask-to-prefix new file mode 100755 index 00000000..1c4dbeb2 --- /dev/null +++ b/bin/netmask-to-prefix @@ -0,0 +1,12 @@ +#! /bin/sh +set -euf + +netmask=$1 + +binaryNetmask=$(echo $1 | sed 's/^/obase=2;/;s/\./;/g' | bc | tr -d \\n) +binaryPrefix=$(echo $binaryNetmask | sed -n 's/^\(1*\)0*$/\1/p') +if ! echo $binaryPrefix | grep -q .; then + echo $0: bad netmask: $netmask >&2 + exit 4 +fi +printf %s $binaryPrefix | tr -d 0 | wc -c diff --git a/bin/nixpkgs-rev b/bin/nixpkgs-rev new file mode 100755 index 00000000..1acde1e4 --- /dev/null +++ b/bin/nixpkgs-rev @@ -0,0 +1,13 @@ +#! /bin/sh +# nixpkgs-rev : nixos-config -> git_rev +set -euf +nix-instantiate \ + -I nixos-config="$1" \ + --eval \ + --json \ + -E \ + ' + (import {config={}; pkgs={};}).nixpkgs.rev + ' \ + 2> /dev/null \ + | jq -r . 2> /dev/null diff --git a/bin/nixpkgs-url b/bin/nixpkgs-url new file mode 100755 index 00000000..9549f0c7 --- /dev/null +++ b/bin/nixpkgs-url @@ -0,0 +1,13 @@ +#! /bin/sh +# nixpkgs-url : nixos-config -> git_url +set -euf +nix-instantiate \ + -I nixos-config="$1" \ + --eval \ + --json \ + -E \ + ' + (import {config={}; pkgs={};}).nixpkgs.url + ' \ + 2> /dev/null \ + | jq -r . 2> /dev/null diff --git a/bin/quoted-strings b/bin/quoted-strings new file mode 100755 index 00000000..e6403910 --- /dev/null +++ b/bin/quoted-strings @@ -0,0 +1,15 @@ +#! /bin/sh +# quoted_strings : lines string |> lines string +# Extract all (double-) quoted strings from stdin. +# +# 0. find begin of string or skip line +# 1. find end of string or skip line +# 2. print string and continue after string +set -euf + +sed ' + s:[^"]*":: ;t1;d + :1; s:\(\([^"]\|\\"\)*\)":\1\n: ;t2;d + :2; P;D +' \ + | sed 's:\\":":g' diff --git a/bin/slash-path-relpath b/bin/slash-path-relpath new file mode 100755 index 00000000..40230a70 --- /dev/null +++ b/bin/slash-path-relpath @@ -0,0 +1,8 @@ +#! /bin/sh +# slash_path_relpath : lines (path ":" relpath) |> lines path +# +# Example: "/foo/bar: baz" => "/foo/baz" +# +set -euf + +sed -n 's@/[^/]\+:@/@p' diff --git a/bin/ssh-deploy b/bin/ssh-deploy new file mode 100755 index 00000000..fe50677d --- /dev/null +++ b/bin/ssh-deploy @@ -0,0 +1,26 @@ +#! /bin/sh +# ssh-deploy : nixos-config x [user@]hostname -> () +set -xeuf + +main=$1 +target=$2 +nixpkgs_dir=/var/nixpkgs # TODO make configurable + +git_url=$(nixpkgs-url $main) +git_rev=$(nixpkgs-rev $main) + +if [ "$git_url" = '' ] || [ "$git_rev" = '' ]; then + echo "specify nixpkgs.url and nixpkgs.rev in $main !" + exit 23 +fi + +filter=$(make-rsync-filter "$main") + +echo "$filter" \ + | rsync -f '. -' -zvrlptD --delete-excluded ./ "$target":/etc/nixos/ + +ssh-fetch-git "$target" "$nixpkgs_dir" "$git_url" "$git_rev" +ssh "$target" nixos-rebuild switch \ + -I nixos-config=/etc/nixos/"$main" \ + -I nixpkgs="$nixpkgs_dir" \ + -I secrets=/etc/nixos/secrets \ diff --git a/bin/ssh-fetch-git b/bin/ssh-fetch-git new file mode 100755 index 00000000..7de58ab7 --- /dev/null +++ b/bin/ssh-fetch-git @@ -0,0 +1,35 @@ +#! /bin/sh +# ssh-fetch-git : [user@]hostname x remote_dir x git_url x git_rev -> () +set -euf + +target=$1 +remote_dir=$2 +git_url=$3 +git_rev=$4 + +echo ' + set -euf + + if [ ! -d "$remote_dir" ]; then + mkdir -p "$remote_dir" + fi + + cd "$remote_dir" + + git init -q + + if ! current_url=$(git config remote.src.url); then + git remote add src "$git_url" + elif [ $current_url != $git_url ]; then + git remote set-url src "$git_url" + fi + + git fetch src + + git checkout "$git_rev" +' \ + | ssh "$target" env \ + remote_dir="$remote_dir" \ + git_rev="$git_rev" \ + git_url="$git_url" \ + /bin/sh diff --git a/bin/undot-paths b/bin/undot-paths new file mode 100755 index 00000000..2ed86bde --- /dev/null +++ b/bin/undot-paths @@ -0,0 +1,14 @@ +#! /bin/sh +# undot_paths : lines path |> lines path +# Remove all dots (. and ..) from input paths. +set -euf + +sed ' + :0 + s://\+:/:g + s:/\.\(/\|$\):\1:g + s:/[^/]\+/\.\.\(/\|$\):\1:g + s:^/\(\.\./\)\+:/: + t0 + s:^$:/: +' diff --git a/bin/urlencode b/bin/urlencode new file mode 100755 index 00000000..02ca0307 --- /dev/null +++ b/bin/urlencode @@ -0,0 +1,35 @@ +#! /bin/sh +set -euf +exec sed ' + s/%/%25/g + s/ /%20/g + s/!/%21/g + s/"/%22/g + s/#/%23/g + s/\$/%24/g + s/\&/%26/g + s/'\''/%27/g + s/(/%28/g + s/)/%29/g + s/\*/%2a/g + s/+/%2b/g + s/,/%2c/g + s/-/%2d/g + s/\./%2e/g + s/\//%2f/g + s/:/%3a/g + s/;/%3b/g + s//%3e/g + s/?/%3f/g + s/@/%40/g + s/\[/%5b/g + s/\\/%5c/g + s/\]/%5d/g + s/\^/%5e/g + s/_/%5f/g + s/`/%60/g + s/{/%7b/g + s/|/%7c/g + s/}/%7d/g + s/~/%7e/g +' diff --git a/deploy b/deploy index 03a20017..43021d54 100755 --- a/deploy +++ b/deploy @@ -4,7 +4,8 @@ # set -euf -. ./lib/prelude.sh +PATH="$PWD/bin${PATH+:$PATH}" +export PATH user=root host=$1 @@ -12,4 +13,4 @@ host=$1 config=./modules/$host/default.nix target=${2-$user@$host} -verbose deploy "$config" "$target" +exec ssh-deploy "$config" "$target" diff --git a/infest b/infest index ca37f49e..8c891c42 100755 --- a/infest +++ b/infest @@ -1,187 +1,15 @@ #! /bin/sh -set -xeuf +# +# usage: ./infest cac-servername hostname +# +set -euf -. ./lib/prelude.sh -. ./lib/cac.sh -. ./lib/cacnixos.sh +PATH="$PWD/bin${PATH+:$PATH}" +export PATH nix_url=https://nixos.org/releases/nix/nix-1.8/nix-1.8-x86_64-linux.tar.bz2 nix_sha256=52fab207b4ce4d098a12d85357d0353e972c492bab0aa9e08e1600363e76fefb nix_find_sha1sum=86f8775bd4f0841edd4c816df861cebf509d58c3 +export nix_url nix_sha256 nix_find_sha1sum -# This is somewhat required because cloudatcost requires whitelisting -# of hosts. If you whitelist your localhost, then leave this empty. -# cac_via= -# -# cac_key= -# cac_login= -# cac_servername= - -# hostname= - -main() { - server=$(cac_getserver_by_servername "$cac_servername") - - serverstatus=$(echo $server | jq -r .status) - case $serverstatus in - 'Powered On') : ;; - *) - echo $0: bad server status: $serverstatus >&2 - exit 2 - esac - - template=$(echo $server | jq -r .template) - case $template in - 'CentOS-7-64bit') infest_centos7_64bit "$server";; - *) - echo $0: bad template: $template >&2 - exit 3 - esac -} - - -infest_centos7_64bit() { - server=$1 - address=$(echo $server | jq -r .ip) - RSYNC_RSH='sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' - SSHPASS=$(echo $server | jq -r .rootpass) - export SSHPASS - export RSYNC_RSH - - main="modules/$hostname/default.nix" - target="root@$address" - - cacnixos_networking "$server" $hostname \ - > modules/$hostname/networking.nix - - echo '( - set -xeuf - type bzip2 || yum install -y bzip2 - type rsync || yum install -y rsync - )' \ - | sshpass -e ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - "root@$address" \ - /bin/sh - - rsync_filter "$main" \ - | rsync -f '. -' -zvrlptD --delete-excluded ./ "$target":/etc/nixos/ - - # - # - # - echo '( - set -xeuf - groupadd -g 30000 nixbld || : - for i in `seq 1 10`; do - useradd -c "foolsgarden Nix build user $i" \ - -d /var/empty \ - -s /sbin/nologin \ - -g 30000 \ - -G 30000 \ - -l -u $(expr 30000 + $i) \ - nixbld$i || : - rm -f /var/spool/mail/nixbld$i - done - - #curl https://nixos.org/nix/install | sh - nix_tar=$nix_basename.tar.bz2 - if ! echo $nix_sha256 $nix_tar | sha256sum -c; then - curl -O -C - $nix_url || : - if ! echo $nix_sha256 $nix_tar | sha256sum -c; then - curl -O $nix_url || : - if ! echo $nix_sha256 $nix_tar | sha256sum -c; then - echo $0: cannot download $nix_url >&2 - exit 5 - fi - fi - fi - - if ! test -d $nix_basename; then - tar jxf $nix_basename.tar.bz2 - fi - - nix_find=$nix_basename.find.txt - if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then - find $nix_basename | sort > $nix_find - if ! echo $nix_find_sha1sum $nix_find | sha1sum -c; then - echo $0: cannot unpack $nix_basename.tar.bz2 >&2 - # TODO we could retry - exit 6 - fi - fi - - mkdir -p bin - PATH=$HOME/bin:$PATH - export PATH - - # generate fake sudo because - # sudo: sorry, you must have a tty to run sudo - { - echo "#! /bin/sh" - echo "exec env \"\$@\"" - } > bin/sudo - chmod +x bin/sudo - - ./$nix_basename/install - - . /root/.nix-profile/etc/profile.d/nix.sh - - nixpkgs_expr="import { system = builtins.currentSystem; }" - nixpkgs_path=$( - find /nix/store -mindepth 1 -maxdepth 1 -name *-nixpkgs-* -type d - ) - - for i in nixos-generate-config nixos-install; do - nix-env \ - --arg config "{ nix.package = ($nixpkgs_expr).nix; }" \ - --arg pkgs "$nixpkgs_expr" \ - --arg modulesPath "throw \"no modulesPath\"" \ - -f $nixpkgs_path/nixpkgs/nixos/modules/installer/tools/tools.nix \ - -iA config.system.build.$i - done - - # TODO following fail when aborted in-between - if ! test -d /int; then - mkdir -p /int - mount --bind /int /mnt - fi - if ! test -d /mnt/boot; then - mkdir -p /mnt/boot - mount /dev/sda1 /mnt/boot - fi - - mkdir -p /mnt/etc/nixos - rsync -zvrlptD --delete-excluded /etc/nixos/ /mnt/etc/nixos/ - - mkdir -m 0444 -p /mnt/var/empty - - ln -s $main /mnt/etc/nixos/configuration.nix - nixos-install \ - -I secrets=/etc/nixos/secrets - - rsync -va --force /int/ / - - # find / -type f -mtime +1 -exec rm -v {} \; 2>&1 > rm.log - # ^ too aggressive, kills journal which is bad - # shutdown -r now - # nix-channel --add https://nixos.org/channels/nixos-unstable nixos - # nix-channel --remove nixpkgs - # nix-channel --update - - )' \ - | sshpass -e ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - "root@$address" \ - -T /usr/bin/env \ - nix_url="$nix_url" \ - nix_basename="$(basename $nix_url .tar.bz2)" \ - nix_sha256="$nix_sha256" \ - nix_find_sha1sum="$nix_find_sha1sum" \ - main="$main" \ - /bin/sh -} - -main "$@" +exec infest-cac "$@" diff --git a/lib/cac.sh b/lib/cac.sh deleted file mode 100644 index fea6886b..00000000 --- a/lib/cac.sh +++ /dev/null @@ -1,105 +0,0 @@ -. ./lib/url.sh - -cac_ssh() {( - server=$1 - shift - - address=$(echo $server | jq -r .ip) - target=root@$address - - SSHPASS=$(echo $server | jq -r .rootpass) - export SSHPASS - - exec sshpass -e ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - "$target" \ - "$@" -)} - -cac_getserver_by_servername() {( - serverlist=$(cac_listservers) - echo $serverlist \ - | jq \ - --arg name "$1" \ - '.[]|select(.servername==$name)' -)} - - -cac_listservers() {( - listservers=$(_cac_get_api_v1 listservers) - status=$(echo "$listservers" | jq -r .status) - if [ "$status" = ok ]; then - echo "$listservers" | jq -r .data - else - echo "$0: bad listservers status: $status" >&2 - exit 1 - fi -)} - -cac_listtasks() { - _cac_get_api_v1 listtasks -} - -cac_listtemplates() { - _cac_get_api_v1 listtemplates -} - -cac_console() { - _cac_post_api_v1 console sid="$1" -} - -cac_powerop() { - _cac_post_api_v1 powerop sid="$1" action="$2" -} - -cac_renameserver() { - _cac_post_api_v1 renameserver sid="$1" name="$2" -} - -cac_rnds() { - _cac_post_api_v1 rdns sid="$1" hostname="$2" -} - -cac_runmode() { - _cac_post_api_v1 rdns sid="$1" mode="$2" -} - -# default os=26 is CentOS-7-64bit -cac_cloudpro_build() { - _cac_post_api_v1 cloudpro/build cpu="$1" ram="$2" storage="$3" os="${4-26}" -} - -cac_cloudpro_delete() { - _cac_post_api_v1 cloudpro/delete sid="$1" -} - -cac_cloudpro_resources() { - _cac_get_api_v1 cloudpro/resources -} - -_cac_get_api_v1() { - _cac_curl_api_v1 -G "$@" -} - -_cac_post_api_v1() { - _cac_curl_api_v1 -XPOST "$@" -} - -_cac_curl_api_v1() { - _cac_exec curl -fsS "$1" "https://panel.cloudatcost.com/api/v1/$2.php" $( - shift 2 - set -- "$@" login="$cac_login" key="$cac_key" - for arg; do - echo -d $(printf '%s' "$arg" | url_encode) - done - ) -} - -_cac_exec() { - if test -z "${cac_via-}"; then - (exec "$@") - else - ssh -q "$cac_via" -t "$@" - fi -} diff --git a/lib/cacnixos.sh b/lib/cacnixos.sh deleted file mode 100644 index 24502d69..00000000 --- a/lib/cacnixos.sh +++ /dev/null @@ -1,28 +0,0 @@ -. ./lib/net.sh - -# cacnixos_networking : cac-server x hostname -> nixos-module -cacnixos_networking() {( - server=$1 - hostname=$2 - - address=$(echo $server | jq -r .ip) - gateway=$(echo $server | jq -r .gateway) - nameserver=8.8.8.8 - netmask=$(echo $server | jq -r .netmask) - prefix=$(net_netmask_to_prefix $netmask) - - printf '{...}:\n' - printf '{\n' - printf ' networking.hostName = "%s";\n' $hostname - printf ' networking.interfaces.enp2s1.ip4 = [\n' - printf ' {\n' - printf ' address = "%s";\n' $address - printf ' prefixLength = %d;\n' $prefix - printf ' }\n' - printf ' ];\n' - printf ' networking.defaultGateway = "%s";\n' $gateway - printf ' networking.nameservers = [\n' - printf ' "%s"\n' $nameserver - printf ' ];\n' - printf '}\n' -)} diff --git a/lib/net.sh b/lib/net.sh deleted file mode 100644 index 518c955b..00000000 --- a/lib/net.sh +++ /dev/null @@ -1,9 +0,0 @@ -net_netmask_to_prefix() {( - binaryNetmask=$(echo $1 | sed 's/^/obase=2;/;s/\./;/g' | bc | tr -d \\n) - binaryPrefix=$(echo $binaryNetmask | sed -n 's/^\(1*\)0*$/\1/p') - if ! echo $binaryPrefix | grep -q .; then - echo $0: bad netmask: $netmask >&2 - exit 4 - fi - printf %s $binaryPrefix | tr -d 0 | wc -c -)} diff --git a/lib/prelude.sh b/lib/prelude.sh deleted file mode 100644 index 2adfb565..00000000 --- a/lib/prelude.sh +++ /dev/null @@ -1,261 +0,0 @@ -# clone_or_update : [user@]hostname x local_dir x git_url x git_rev -> () -clone_or_update() {( - target=$1 - nixpkgs_dir=$2 - git_url=$3 - git_rev=$4 - - echo ' - set -euf - - if [ ! -d "$nixpkgs_dir" ]; then - mkdir -p "$nixpkgs_dir" - fi - - cd "$nixpkgs_dir" - - git init -q - - if ! current_url=$(git config remote.src.url); then - git remote add src "$git_url" - elif [ $current_url != $git_url ]; then - git remote set-url src "$git_url" - fi - - git fetch src - - git checkout "$git_rev" - ' \ - | ssh "$target" env \ - nixpkgs_dir="$nixpkgs_dir" \ - git_rev="$git_rev" \ - git_url="$git_url" \ - /bin/sh -)} - -# deploy : nixos-config x [user@]hostname -> () -deploy() {( - main=$1 - target=$2 - nixpkgs_dir=/var/nixpkgs # TODO make configurable - - git_url=$(nixpkgs_url $main) - git_rev=$(nixpkgs_rev $main) - - if [ "$git_url" = '' ] || [ "$git_rev" = '' ]; then - echo "specify nixpkgs.url and nixpkgs.rev in $main !" - exit 23 - fi - - filter=$(rsync_filter "$main") - - echo "$filter" \ - | rsync -f '. -' -zvrlptD --delete-excluded ./ "$target":/etc/nixos/ - - clone_or_update "$target" "$nixpkgs_dir" "$git_url" "$git_rev" - ssh "$target" nixos-rebuild switch \ - -I nixos-config=/etc/nixos/"$main" \ - -I nixpkgs="$nixpkgs_dir" \ - -I secrets=/etc/nixos/secrets \ -)} - -# rsync_filter : nixos-config -> rsync-filter -rsync_filter() {( - main=$1 - - hosts=$(list_hosts) - module_imports=$(set -euf; list_module_imports "$main") - other_imports=$( - echo "$module_imports" \ - | xargs grep -H . \ - | import_statements \ - | slash_path_relpath \ - | undot_paths \ - | sort \ - | uniq \ - | sed '/\.nix$/!s:$:/default.nix:' \ - ) - secrets=$(echo "$module_imports" | xargs cat | quoted_strings | filter_secrets) - - # TODO collect all other paths from *_imports - - abs_deps=$( - echo "$hosts" - echo "$module_imports" - echo "$other_imports" - echo "$secrets" - ) - - rel_deps=$(echo "$abs_deps" | make_relative_to "$PWD") - filter=$(echo "$rel_deps" | make_rsync_whitelist) - - echo "$filter" -)} - -# list_module_imports : nix-file -> lines nix-file -list_module_imports() { - if echo "$1" | grep -q ^/; then - : - else - set -- "./$1" - fi - imports=$(nix-instantiate \ - -I secrets=secrets \ - --strict \ - --json \ - --eval \ - -E \ - "with builtins; with import ./lib/modules.nix; map toString (list-imports $1)") - echo "$imports" \ - | jq -r .[] -} - -# list_hosts : lines tinc-host-file -# Precondition: $PWD/hosts is the correct repository :) -list_hosts() { - git -C hosts ls-tree --name-only HEAD \ - | awk '{print ENVIRON["PWD"]"/hosts/"$$0}' -} - -# filter_secrets : lines string |> lines secrets-file-candidate -# Notice how false positives are possible. -filter_secrets() { - sed -n 's:^\(.*/\)\?\(secrets/.*\):'"${PWD//:/\\:}"'/\2:p' -} - -# import_statements : lines (path ":" string) |> lines (path ":" relpath) -import_statements() { - sed -n ' - s@^\([^:]\+:\)\('"$(bre_invert_word import)"'\)*\ lines path -# -# Example: "/foo/bar: baz" => "/foo/baz" -# -slash_path_relpath() { - sed -n 's@/[^/]\+:@/@p' -} - -# undot_paths : lines path |> lines path -# Remove all dots (. and ..) from input paths. -undot_paths() { - sed ' - :0 - s://\+:/:g - s:/\.\(/\|$\):\1:g - s:/[^/]\+/\.\.\(/\|$\):\1:g - s:^/\(\.\./\)\+:/: - t0 - s:^$:/: - ' -} - -# quoted_strings : lines string |> lines string -# Extract all (double-) quoted strings from stdin. -# -# 0. find begin of string or skip line -# 1. find end of string or skip line -# 2. print string and continue after string -quoted_strings() { - sed ' - s:[^"]*":: ;t1;d - :1; s:\(\([^"]\|\\"\)*\)":\1\n: ;t2;d - :2; P;D - ' \ - | sed 's:\\":":g' -} - -# bre_escape : lines string |> lines bre-escaped-string -bre_escape() { - sed 's:[\.\[\\\*\^\$]:\\&:g' -} - -# bre_invert_word : string -> BRE -# TODO escape chars in the resulting BRE. -bre_invert_word() { - awk -v input="$1" ' - BEGIN { - split(input,s,"") - for (i in s) { - c=s[i] - printf "\\|%s[^%s]", y, c - y = y c - } - } - ' -} - -# ls_bre : directory -> BRE -# Create a BRE from the files in a directory. -ls_bre() { - ls "$1" \ - | tr \\n / \ - | sed ' - s:[\.\[\\\*\^\$]:\\&:g - s:/$:: - s:/:\\|:g - ' -} - -# make_relative_to : lines path |> directory -> lines path -# Non-matching paths won't get altered. -make_relative_to() { - sed "s:^$(echo "$1/" | bre_escape | sed 's/:/\\:/g')::" -} - -# make_rsync_whitelist : lines relpath |> liens rsync-filter -make_rsync_whitelist() { - set -- "$(cat)" - - # include all files in stdin and their directories - { - echo "$1" - echo "$1" | make_parent_dirs | sort | uniq - } \ - | sed 's|^|+ /|' - - # exclude everything else - echo '- *' -} - -# make_parent_dirs : lines path |> lines directory -# List all parent directories of a path. -make_parent_dirs() { - set -- "$(sed -n 's|/[^/]*$||p' | grep . | sort | uniq)" - if echo "$1" | grep -q .; then - echo "$1" - echo "$1" | make_parent_dirs - fi -} - -# nixpkgs_url : nixos-config -> git_url -nixpkgs_url() { - nix-instantiate \ - -I nixos-config="$1" \ - --eval \ - --json \ - -E '(import {config={}; pkgs={};}).nixpkgs.url' 2> /dev/null \ - | jq -r . -} - -# nixpkgs_rev : nixos-config -> git_rev -nixpkgs_rev() { - nix-instantiate \ - -I nixos-config="$1" \ - --eval \ - --json \ - -E '(import {config={}; pkgs={};}).nixpkgs.rev' 2> /dev/null \ - | jq -r . 2> /dev/null -} - -# verbose COMMAND [ARGS...] -verbose() { - echo "$@" >&2 - "$@" -} diff --git a/lib/url.sh b/lib/url.sh deleted file mode 100644 index 05f93a94..00000000 --- a/lib/url.sh +++ /dev/null @@ -1,35 +0,0 @@ -url_encode() { - sed ' - s/%/%25/g - s/ /%20/g - s/!/%21/g - s/"/%22/g - s/#/%23/g - s/\$/%24/g - s/\&/%26/g - s/'\''/%27/g - s/(/%28/g - s/)/%29/g - s/\*/%2a/g - s/+/%2b/g - s/,/%2c/g - s/-/%2d/g - s/\./%2e/g - s/\//%2f/g - s/:/%3a/g - s/;/%3b/g - s//%3e/g - s/?/%3f/g - s/@/%40/g - s/\[/%5b/g - s/\\/%5c/g - s/\]/%5d/g - s/\^/%5e/g - s/_/%5f/g - s/`/%60/g - s/{/%7b/g - s/|/%7c/g - s/}/%7d/g - s/~/%7e/g - ' -} -- cgit v1.2.3