From 4630d10b3151f689247c0e8e7488917ee6313c7f Mon Sep 17 00:00:00 2001 From: tv Date: Fri, 17 May 2019 12:50:48 +0200 Subject: github-hosts-sync: import 1.0.0 from painload --- krebs/5pkgs/simple/github-hosts-sync/default.nix | 4 +- .../5pkgs/simple/github-hosts-sync/src/hosts-sync | 66 ++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100755 krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync (limited to 'krebs') diff --git a/krebs/5pkgs/simple/github-hosts-sync/default.nix b/krebs/5pkgs/simple/github-hosts-sync/default.nix index cdfed468c..8caa5e1ef 100644 --- a/krebs/5pkgs/simple/github-hosts-sync/default.nix +++ b/krebs/5pkgs/simple/github-hosts-sync/default.nix @@ -3,7 +3,7 @@ stdenv.mkDerivation { name = "github-hosts-sync"; - src = pkgs.painload; + src = ./src; phases = [ "unpackPhase" @@ -29,7 +29,7 @@ stdenv.mkDerivation { sed \ 's,^main() {$,&\n export PATH=${path} GIT_SSL_CAINFO=${ca-bundle},' \ - < ./retiolum/scripts/github_hosts_sync/hosts-sync \ + < hosts-sync \ > $out/bin/github-hosts-sync chmod +x $out/bin/github-hosts-sync diff --git a/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync b/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync new file mode 100755 index 000000000..f36c700d8 --- /dev/null +++ b/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync @@ -0,0 +1,66 @@ +#! /bin/sh +# TODO do_work should retry until success +set -euf + +port=${port-1028} +local_painload=$HOME/painload +remote_painload="https://github.com/krebscode/painload" +local_hosts=$HOME/hosts +remote_hosts="git@github.com:krebscode/hosts.git" + +main() { + ensure_local_painload + ensure_local_hosts + is_worker && do_work || become_server +} + +ensure_local_painload() { + test -d "$local_painload" || git clone "$remote_painload" "$local_painload" +} + +ensure_local_hosts() { + test -d "$local_hosts" || git clone "$remote_hosts" "$local_hosts" +} + +become_server() { + exec socat "TCP-LISTEN:$port,reuseaddr,fork" "EXEC:$0" +} + +is_worker() { + test "${SOCAT_SOCKPORT-}" = "$port" +} + +do_work() { + # read request + req_line="$(read line && echo "$line")" + req_hdrs="$(sed -n '/^\r$/q;p')" + + set -x + + cd "$local_hosts" + git pull >&2 + + cd "$local_hosts" + find . -name .git -prune -o -type f -exec git rm \{\} \; >/dev/null + + cd "$local_painload" + git pull >&2 + + find "$local_painload/retiolum/hosts" -type f -exec cp \{\} "$local_hosts" \; + + cd "$local_hosts" + find . -name .git -prune -o -type f -exec git add \{\} \; >&2 + if git status --porcelain | grep -q .; then + git config user.email "$LOGNAME@$(hostname)" + git config user.name "$LOGNAME" + git commit -m bump >&2 + git push >&2 + fi + + echo "HTTP/1.1 200 OK" + echo + echo "https://github.com/krebscode/hosts/archive/master.tar.gz" + echo "https://github.com/krebscode/hosts/archive/master.zip" +} + +main "$@" -- cgit v1.2.3 From acb3f95fa6586a9c9b1b1ffa76368c1b39edb8aa Mon Sep 17 00:00:00 2001 From: tv Date: Fri, 17 May 2019 13:06:36 +0200 Subject: github-hosts-sync: 1.0.0 -> 2.0.0 --- krebs/3modules/github-hosts-sync.nix | 25 +++++-- krebs/5pkgs/simple/github-hosts-sync/default.nix | 36 ++++------ .../5pkgs/simple/github-hosts-sync/src/hosts-sync | 81 ++++++---------------- 3 files changed, 56 insertions(+), 86 deletions(-) (limited to 'krebs') diff --git a/krebs/3modules/github-hosts-sync.nix b/krebs/3modules/github-hosts-sync.nix index 3b626dc46..233cea68d 100644 --- a/krebs/3modules/github-hosts-sync.nix +++ b/krebs/3modules/github-hosts-sync.nix @@ -11,17 +11,25 @@ let api = { enable = mkEnableOption "krebs.github-hosts-sync"; - port = mkOption { - type = types.int; # TODO port type - default = 1028; - }; dataDir = mkOption { type = types.str; # TODO path (but not just into store) default = "/var/lib/github-hosts-sync"; }; + srcDir = mkOption { + type = types.str; + default = "${config.krebs.tinc.retiolum.confDir}/hosts"; + }; ssh-identity-file = mkOption { type = types.suffixed-str [".ssh.id_ed25519" ".ssh.id_rsa"]; - default = toString ; + default = toString ; + }; + url = mkOption { + type = types.str; + default = "git@github.com:krebscode/hosts.git"; + }; + workTree = mkOption { + type = types.absolute-pathname; + default = "${cfg.dataDir}/cache"; }; }; @@ -30,13 +38,16 @@ let after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; environment = { - port = toString cfg.port; + GITHUB_HOST_SYNC_SRCDIR = cfg.srcDir; + GITHUB_HOST_SYNC_WORKTREE = cfg.workTree; + GITHUB_HOST_SYNC_URL = cfg.url; }; serviceConfig = { PermissionsStartOnly = "true"; SyslogIdentifier = "github-hosts-sync"; User = user.name; - Restart = "always"; + Type = "oneshot"; + RemainAfterExit = true; ExecStartPre = pkgs.writeDash "github-hosts-sync-init" '' set -euf install -m 0711 -o ${user.name} -d ${cfg.dataDir} diff --git a/krebs/5pkgs/simple/github-hosts-sync/default.nix b/krebs/5pkgs/simple/github-hosts-sync/default.nix index 8caa5e1ef..5caf225cb 100644 --- a/krebs/5pkgs/simple/github-hosts-sync/default.nix +++ b/krebs/5pkgs/simple/github-hosts-sync/default.nix @@ -1,7 +1,8 @@ { pkgs, stdenv, ... }: -stdenv.mkDerivation { - name = "github-hosts-sync"; +stdenv.mkDerivation rec { + name = "github-hosts-sync-${version}"; + version = "2.0.0"; src = ./src; @@ -10,28 +11,21 @@ stdenv.mkDerivation { "installPhase" ]; - installPhase = - let - ca-bundle = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; - path = stdenv.lib.makeBinPath (with pkgs; [ - coreutils - findutils - git - gnugrep - gnused - nettools - openssh - socat - ]); - in + installPhase = let + ca-bundle = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + path = stdenv.lib.makeBinPath [ + pkgs.git + pkgs.openssh + pkgs.rsync + ]; + in '' mkdir -p $out/bin - sed \ - 's,^main() {$,&\n export PATH=${path} GIT_SSL_CAINFO=${ca-bundle},' \ - < hosts-sync \ - > $out/bin/github-hosts-sync + cp hosts-sync $out/bin/github-hosts-sync - chmod +x $out/bin/github-hosts-sync + sed -i \ + '1s,$,\nPATH=${path}''${PATH+:$PATH} GIT_SSL_CAINFO=${ca-bundle},' \ + $out/bin/github-hosts-sync ''; } diff --git a/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync b/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync index f36c700d8..4bae44bef 100755 --- a/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync +++ b/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync @@ -1,66 +1,31 @@ #! /bin/sh -# TODO do_work should retry until success -set -euf +set -efu +exec >&2 -port=${port-1028} -local_painload=$HOME/painload -remote_painload="https://github.com/krebscode/painload" -local_hosts=$HOME/hosts -remote_hosts="git@github.com:krebscode/hosts.git" +hosts_srcdir=$GITHUB_HOST_SYNC_SRCDIR +hosts_worktree=${GITHUB_HOST_SYNC_WORKTREE-/tmp/hosts} +hosts_url=${GITHUB_HOST_SYNC_URL-git@github.com:krebscode/hosts.git} -main() { - ensure_local_painload - ensure_local_hosts - is_worker && do_work || become_server -} +test -d "$hosts_worktree" || git clone "$hosts_url" "$hosts_worktree" -ensure_local_painload() { - test -d "$local_painload" || git clone "$remote_painload" "$local_painload" -} +cd "$hosts_worktree" -ensure_local_hosts() { - test -d "$local_hosts" || git clone "$remote_hosts" "$local_hosts" -} +git pull -become_server() { - exec socat "TCP-LISTEN:$port,reuseaddr,fork" "EXEC:$0" -} +rsync \ + --chmod D755,F644 \ + --delete-excluded \ + --filter 'protect .git' \ + --recursive \ + --verbose \ + "$hosts_srcdir/" \ + . -is_worker() { - test "${SOCAT_SOCKPORT-}" = "$port" -} +git add . -do_work() { - # read request - req_line="$(read line && echo "$line")" - req_hdrs="$(sed -n '/^\r$/q;p')" - - set -x - - cd "$local_hosts" - git pull >&2 - - cd "$local_hosts" - find . -name .git -prune -o -type f -exec git rm \{\} \; >/dev/null - - cd "$local_painload" - git pull >&2 - - find "$local_painload/retiolum/hosts" -type f -exec cp \{\} "$local_hosts" \; - - cd "$local_hosts" - find . -name .git -prune -o -type f -exec git add \{\} \; >&2 - if git status --porcelain | grep -q .; then - git config user.email "$LOGNAME@$(hostname)" - git config user.name "$LOGNAME" - git commit -m bump >&2 - git push >&2 - fi - - echo "HTTP/1.1 200 OK" - echo - echo "https://github.com/krebscode/hosts/archive/master.tar.gz" - echo "https://github.com/krebscode/hosts/archive/master.zip" -} - -main "$@" +if test -n "$(git status --porcelain)"; then + git config user.email "$LOGNAME@$(hostname)" + git config user.name "$LOGNAME" + git commit -m bump + git push +fi -- cgit v1.2.3 From 866e94b4fa70181b9ae753b51d59c27ce42c9497 Mon Sep 17 00:00:00 2001 From: tv Date: Fri, 17 May 2019 13:36:13 +0200 Subject: hotdog.r: enable github-hosts-sync --- krebs/1systems/hotdog/config.nix | 1 + 1 file changed, 1 insertion(+) (limited to 'krebs') diff --git a/krebs/1systems/hotdog/config.nix b/krebs/1systems/hotdog/config.nix index f68c8ce50..32e416831 100644 --- a/krebs/1systems/hotdog/config.nix +++ b/krebs/1systems/hotdog/config.nix @@ -18,6 +18,7 @@ ]; krebs.build.host = config.krebs.hosts.hotdog; + krebs.github-hosts-sync.enable = true; boot.isContainer = true; networking.useDHCP = false; -- cgit v1.2.3 From c7cfc7d6a3988615fd40369d0e02bd570a52bc7f Mon Sep 17 00:00:00 2001 From: tv Date: Fri, 17 May 2019 13:43:13 +0200 Subject: github-hosts-sync: update default URL --- krebs/3modules/github-hosts-sync.nix | 2 +- krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'krebs') diff --git a/krebs/3modules/github-hosts-sync.nix b/krebs/3modules/github-hosts-sync.nix index 233cea68d..6ffaf5503 100644 --- a/krebs/3modules/github-hosts-sync.nix +++ b/krebs/3modules/github-hosts-sync.nix @@ -25,7 +25,7 @@ let }; url = mkOption { type = types.str; - default = "git@github.com:krebscode/hosts.git"; + default = "git@github.com:krebs/hosts.git"; }; workTree = mkOption { type = types.absolute-pathname; diff --git a/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync b/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync index 4bae44bef..d2017ef63 100755 --- a/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync +++ b/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync @@ -4,7 +4,7 @@ exec >&2 hosts_srcdir=$GITHUB_HOST_SYNC_SRCDIR hosts_worktree=${GITHUB_HOST_SYNC_WORKTREE-/tmp/hosts} -hosts_url=${GITHUB_HOST_SYNC_URL-git@github.com:krebscode/hosts.git} +hosts_url=${GITHUB_HOST_SYNC_URL-git@github.com:krebs/hosts.git} test -d "$hosts_worktree" || git clone "$hosts_url" "$hosts_worktree" -- cgit v1.2.3 From e91f56a4092b47aea6dd62e015176c0a45b6e0e6 Mon Sep 17 00:00:00 2001 From: tv Date: Fri, 17 May 2019 13:48:48 +0200 Subject: krebs: add dummy github-hosts-sync.ssh.id_ed25519 --- krebs/0tests/data/secrets/github-hosts-sync.ssh.id_ed25519 | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 krebs/0tests/data/secrets/github-hosts-sync.ssh.id_ed25519 (limited to 'krebs') diff --git a/krebs/0tests/data/secrets/github-hosts-sync.ssh.id_ed25519 b/krebs/0tests/data/secrets/github-hosts-sync.ssh.id_ed25519 new file mode 100644 index 000000000..e69de29bb -- cgit v1.2.3 From 2950b893b03253ef8000e939915bb9c8c1f1f524 Mon Sep 17 00:00:00 2001 From: tv Date: Fri, 17 May 2019 13:53:55 +0200 Subject: github-hosts-sync: add nettools --- krebs/5pkgs/simple/github-hosts-sync/default.nix | 1 + 1 file changed, 1 insertion(+) (limited to 'krebs') diff --git a/krebs/5pkgs/simple/github-hosts-sync/default.nix b/krebs/5pkgs/simple/github-hosts-sync/default.nix index 5caf225cb..fbc48fa3f 100644 --- a/krebs/5pkgs/simple/github-hosts-sync/default.nix +++ b/krebs/5pkgs/simple/github-hosts-sync/default.nix @@ -15,6 +15,7 @@ stdenv.mkDerivation rec { ca-bundle = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; path = stdenv.lib.makeBinPath [ pkgs.git + pkgs.nettools pkgs.openssh pkgs.rsync ]; -- cgit v1.2.3 From a666abeaabbed73749cd5e2f1745b4a4527c4bc6 Mon Sep 17 00:00:00 2001 From: tv Date: Fri, 17 May 2019 14:02:22 +0200 Subject: github-hosts-sync: make user name/mail overridable --- krebs/3modules/github-hosts-sync.nix | 3 +++ krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'krebs') diff --git a/krebs/3modules/github-hosts-sync.nix b/krebs/3modules/github-hosts-sync.nix index 6ffaf5503..0b7d56098 100644 --- a/krebs/3modules/github-hosts-sync.nix +++ b/krebs/3modules/github-hosts-sync.nix @@ -38,6 +38,8 @@ let after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; environment = { + GITHUB_HOST_SYNC_USER_MAIL = user.mail; + GITHUB_HOST_SYNC_USER_NAME = user.name; GITHUB_HOST_SYNC_SRCDIR = cfg.srcDir; GITHUB_HOST_SYNC_WORKTREE = cfg.workTree; GITHUB_HOST_SYNC_URL = cfg.url; @@ -67,6 +69,7 @@ let }; user = rec { + mail = "${name}@${config.krebs.build.host.name}"; name = "github-hosts-sync"; uid = genid_uint31 name; }; diff --git a/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync b/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync index d2017ef63..a8973e72b 100755 --- a/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync +++ b/krebs/5pkgs/simple/github-hosts-sync/src/hosts-sync @@ -5,6 +5,8 @@ exec >&2 hosts_srcdir=$GITHUB_HOST_SYNC_SRCDIR hosts_worktree=${GITHUB_HOST_SYNC_WORKTREE-/tmp/hosts} hosts_url=${GITHUB_HOST_SYNC_URL-git@github.com:krebs/hosts.git} +user_mail=${GITHUB_HOST_SYNC_USER_MAIL-$LOGNAME@$(hostname)} +user_name=${GITHUB_HOST_SYNC_USER_NAME-$LOGNAME} test -d "$hosts_worktree" || git clone "$hosts_url" "$hosts_worktree" @@ -24,8 +26,8 @@ rsync \ git add . if test -n "$(git status --porcelain)"; then - git config user.email "$LOGNAME@$(hostname)" - git config user.name "$LOGNAME" + git config user.email "$user_mail" + git config user.name "$user_name" git commit -m bump git push fi -- cgit v1.2.3 From eb9c9b80cafbb69d858a9914eda1d5aa65745ae5 Mon Sep 17 00:00:00 2001 From: tv Date: Tue, 21 May 2019 10:39:18 +0200 Subject: github-known-hosts: add new hosts --- krebs/3modules/github-known-hosts.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'krebs') diff --git a/krebs/3modules/github-known-hosts.nix b/krebs/3modules/github-known-hosts.nix index def06f17a..bae8b96bf 100644 --- a/krebs/3modules/github-known-hosts.nix +++ b/krebs/3modules/github-known-hosts.nix @@ -28,12 +28,22 @@ "140.82.125.*" "140.82.126.*" "140.82.127.*" + "13.114.40.48" "13.229.188.59" + "13.234.176.102" + "13.234.210.38" + "13.236.229.21" + "13.237.44.5" "13.250.177.223" + "15.164.81.167" "18.194.104.89" "18.195.85.27" "35.159.8.160" + "52.192.72.89" + "52.64.108.95" + "52.69.186.44" "52.74.223.119" + "52.78.231.108" ]; publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="; }; -- cgit v1.2.3