From 7aa31b7babafb2e10bbf8418800ef561bccbd67b Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 25 Jul 2017 23:10:51 +0200 Subject: krebs buildbot: reintroduce packages with minimal overrides --- krebs/3modules/buildbot/master.nix | 28 ++++++---------------------- krebs/3modules/buildbot/slave.nix | 19 ++----------------- 2 files changed, 8 insertions(+), 39 deletions(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 12c685b8..a7624c8f 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -2,22 +2,6 @@ with import ; let - # https://github.com/NixOS/nixpkgs/issues/14026 - nixpkgs-fix = import (pkgs.fetchgit { - url = https://github.com/nixos/nixpkgs; - rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; - sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8"; - }) {}; - - buildbot = nixpkgs-fix.buildbot.overrideDerivation (old: { - postUnpack = "sourceRoot=\${sourceRoot}/master"; - patches = []; - src = pkgs.fetchFromGitHub { - owner = "krebscode"; - repo = "buildbot-classic"; - rev = "5b4f5f6f1"; - sha256 = "1j3xn1gjzvsf90jvfmyln71fzlhjx642ivrqf47zfxpkacljja93"; };}); - buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' # -*- python -*- from buildbot.plugins import * @@ -364,7 +348,7 @@ let set -efux if [ ! -e ${workdir} ];then mkdir -p ${workdir} - ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir} + ${pkgs.buildbot-classic}/bin/buildbot create-master -r -l 10 -f ${workdir} fi # always override the master.cfg cp ${buildbot-master-config} ${workdir}/master.cfg @@ -373,18 +357,18 @@ let ${ concatMapStringsSep "\n" (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets } # sanity - ${buildbot}/bin/buildbot checkconfig ${workdir} + ${pkgs.buildbot-classic}/bin/buildbot checkconfig ${workdir} # TODO: maybe upgrade? not sure about this # normally we should write buildbot.tac by our own - # ${buildbot}/bin/buildbot upgrade-master ${workdir} + # ${pkgs.buildbot-classic}/bin/buildbot upgrade-master ${workdir} chmod 700 -R ${workdir} chown buildbotMaster:buildbotMaster -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; - ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; - ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + ExecStart = "${pkgs.buildbot-classic}/bin/buildbot start ${workdir}"; + ExecStop = "${pkgs.buildbot-classic}/bin/buildbot stop ${workdir}"; + ExecReload = "${pkgs.buildbot-classic}/bin/buildbot reconfig ${workdir}"; PrivateTmp = "true"; User = "buildbotMaster"; Restart = "always"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 698bf3bc..544f9c4e 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -2,20 +2,6 @@ with import ; let - # https://github.com/NixOS/nixpkgs/issues/14026 - nixpkgs-fix = import (pkgs.fetchgit { - url = https://github.com/nixos/nixpkgs; - rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; - sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8"; - }) {}; - pkg = nixpkgs-fix.buildbot-slave.overrideDerivation (old: { - postUnpack = "sourceRoot=\${sourceRoot}/slave"; - patches = []; - src = pkgs.fetchFromGitHub { - owner = "krebscode"; - repo = "buildbot-classic"; - rev = "5b4f5f6f1"; - sha256 = "1j3xn1gjzvsf90jvfmyln71fzlhjx642ivrqf47zfxpkacljja93"; };}); buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' import os @@ -166,7 +152,6 @@ let workdir = shell.escape cfg.workDir; contact = shell.escape cfg.contact; description = shell.escape cfg.description; - buildbot = pkg; # TODO:make this in { PermissionsStartOnly = true; @@ -183,8 +168,8 @@ let chown buildbotSlave:buildbotSlave -R ${workdir} chmod 700 -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildslave start ${workdir}"; - ExecStop = "${buildbot}/bin/buildslave stop ${workdir}"; + ExecStart = "${pkgs.buildbot-classic-slave}/bin/buildslave start ${workdir}"; + ExecStop = "${pkgs.buildbot-classic-slave}/bin/buildslave stop ${workdir}"; PrivateTmp = "true"; User = "buildbotSlave"; Restart = "always"; -- cgit v1.2.3 From 4c8afbd21041c49e4eb6ed5357c6455391947014 Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 26 Jul 2017 00:21:29 +0200 Subject: krebs: add hotdog-repo-sync --- krebs/3modules/krebs/default.nix | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'krebs/3modules') diff --git a/krebs/3modules/krebs/default.nix b/krebs/3modules/krebs/default.nix index 07543489..0d990bf4 100644 --- a/krebs/3modules/krebs/default.nix +++ b/krebs/3modules/krebs/default.nix @@ -120,6 +120,11 @@ in { krebs = { pubkey = "lol"; # TODO krebs.users.krebs.pubkey should be unnecessary }; + hotdog-repo-sync = { + name = "hotdog-repo-sync"; + mail = "spam@krebsco.de"; + pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILzTvaR3QqOD3oEEGHQzg/sRnNbKJnZYcV9htDvXmu53"; + }; puyak-repo-sync = { name = "puyak-repo-sync"; mail = "spam@krebsco.de"; -- cgit v1.2.3 From 18451c9dff24e42739f97c148360c5a7c06866b5 Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 26 Jul 2017 00:22:45 +0200 Subject: add {cgit,build}.hotdog.r --- krebs/3modules/krebs/default.nix | 2 ++ 1 file changed, 2 insertions(+) (limited to 'krebs/3modules') diff --git a/krebs/3modules/krebs/default.nix b/krebs/3modules/krebs/default.nix index 0d990bf4..20b2384e 100644 --- a/krebs/3modules/krebs/default.nix +++ b/krebs/3modules/krebs/default.nix @@ -38,6 +38,8 @@ in { ip6.addr = "42:0:0:0:0:0:77:3"; aliases = [ "hotdog.r" + "build.hotdog.r" + "cgit.hotdog.r" ]; tinc.pubkey = '' -----BEGIN RSA PUBLIC KEY----- -- cgit v1.2.3 From 1c659508ae476ada65982ff06f34e1d3ccf8ca7d Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 27 Jul 2017 19:43:41 +0200 Subject: krebs: set managed hosts --- krebs/3modules/krebs/default.nix | 3 +++ 1 file changed, 3 insertions(+) (limited to 'krebs/3modules') diff --git a/krebs/3modules/krebs/default.nix b/krebs/3modules/krebs/default.nix index 20b2384e..27fbb708 100644 --- a/krebs/3modules/krebs/default.nix +++ b/krebs/3modules/krebs/default.nix @@ -32,6 +32,7 @@ in { hosts = { hotdog = { owner = config.krebs.users.krebs; + managed = true; nets = { retiolum = { ip4.addr = "10.243.77.3"; @@ -58,6 +59,7 @@ in { }; puyak = { owner = config.krebs.users.krebs; + managed = true; nets = { retiolum = { ip4.addr = "10.243.77.2"; @@ -84,6 +86,7 @@ in { }; wolf = { owner = config.krebs.users.krebs; + managed = true; nets = { shack = { ip4.addr = "10.42.2.150" ; -- cgit v1.2.3 From 2257b79367b5f8015c3367b890f66e97d1b00a43 Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 27 Jul 2017 19:45:45 +0200 Subject: add module: krebs.ci --- krebs/3modules/ci.nix | 175 +++++++++++++++++++++++++++++++++++++++++++++ krebs/3modules/default.nix | 1 + 2 files changed, 176 insertions(+) create mode 100644 krebs/3modules/ci.nix (limited to 'krebs/3modules') diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix new file mode 100644 index 00000000..542a9252 --- /dev/null +++ b/krebs/3modules/ci.nix @@ -0,0 +1,175 @@ +{ config, pkgs, ... }: +with import ; +let + cfg = config.krebs.ci; + + hostname = config.networking.hostName; +in +{ + options.krebs.ci = { + enable = mkEnableOption "krebs continous integration"; + users = mkOption { + type = with types; attrsOf (submodule { + options = { + all = mkOption { + type = bool; + default = true; + }; + hosts = mkOption { + type = listOf str; + default = []; + }; + }; + }); + example = { + lass.all = true; + krebs = { + all = true; + hosts = [ + "test-all-krebs-modules" + "test-arch" + ]; + }; + }; + default = {}; + }; + }; + + config = mkIf cfg.enable { + services.nginx = { + enable = true; + virtualHosts.build = { + serverAliases = [ "build.${hostname}.r" ]; + locations."/".extraConfig = '' + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_pass http://127.0.0.1:${toString config.krebs.buildbot.master.web.port}; + ''; + }; + }; + + nix.gc.automatic = true; + nix.gc.dates = "05:23"; + + krebs.buildbot.master = { + slaves = { + testslave = "lasspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.${hostname}.r/stockholm' + cs.append( + changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branches=True, + project='stockholm', + pollinterval=10 + ) + ) + ''; + scheduler = { + build-scheduler = '' + # build all hosts + sched.append( + schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch_re=".*"), + treeStableTimer=10, + name="build-all-branches", + builderNames=[ + "build-hosts" + ] + ) + ) + ''; + force-scheduler = '' + sched.append( + schedulers.ForceScheduler( + name="force", + builderNames=[ + "build-hosts" + ] + ) + ) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + grab_repo = steps.Git( + repourl=stockholm_repo, + mode='full' + ) + + # prepare addShell function + def addShell(factory,**kwargs): + factory.addStep(steps.ShellCommand(**kwargs)) + ''; + builder = { + build-hosts = '' + f = util.BuildFactory() + f.addStep(grab_repo) + + def build_host(user, host): + addShell(f, + name="{}".format(host), + env={ + "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src", + "NIX_REMOTE": "daemon", + "dummy_secrets": "true", + }, + command=[ + "nix-shell", "--run", + "test --user={} --system={} --target=$LOGNAME@${config.krebs.build.host.name}$HOME/{}".format(user, host, user) + ] + ) + + ${let + user-hosts = mapAttrs (user: a: let + managed-hosts = attrNames (filterAttrs (_: h: (h.owner.name == user) && h.managed) config.krebs.hosts); + defined-hosts = a.hosts; + in + defined-hosts ++ (optionals a.all managed-hosts) + ) cfg.users; + + in + concatStringsSep "\n" ( + (mapAttrsToList (user: hosts: + concatMapStringsSep "\n" (host: + "build_host(\"${user}\", \"${host}\")" + ) hosts + ) user-hosts) + ) + } + + bu.append( + util.BuilderConfig( + name="build-hosts", + slavenames=slavenames, + factory=f + ) + ) + + ''; + }; + enable = true; + web.enable = true; + irc = { + enable = true; + nick = "build|${hostname}"; + server = "ni.r"; + channels = [ "retiolum" "noise" ]; + allowForce = true; + }; + extraConfig = '' + c['buildbotURL'] = "http://build.${hostname}.r/" + ''; + }; + + krebs.buildbot.slave = { + enable = true; + masterhost = "localhost"; + username = "testslave"; + password = "lasspass"; + packages = with pkgs; [ gnumake jq nix populate ]; + }; + + }; +} diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 6123b6dd..b0ad2baf 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -12,6 +12,7 @@ let ./buildbot/master.nix ./buildbot/slave.nix ./build.nix + ./ci.nix ./current.nix ./exim.nix ./exim-retiolum.nix -- cgit v1.2.3 From 38810e7a2988deb6f360f2d7d020e3038c2e510a Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 28 Jul 2017 12:02:29 +0200 Subject: add host: daedalus --- krebs/3modules/lass/default.nix | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'krebs/3modules') diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix index 139f02dd..2e257b8a 100644 --- a/krebs/3modules/lass/default.nix +++ b/krebs/3modules/lass/default.nix @@ -276,6 +276,32 @@ with import ; ssh.privkey.path = ; ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOPgQIMYiyD4/Co+nlOQWEzCKssemOEXAY/lbIZZaMhj"; }; + daedalus = { + cores = 2; + nets = rec { + retiolum = { + ip4.addr = "10.243.133.115"; + ip6.addr = "42:0:0:0:0:0:daed:a105"; + aliases = [ + "daedalus.r" + "cgit.daedalus.r" + ]; + tinc.pubkey = '' + -----BEGIN RSA PUBLIC KEY----- + MIIBCgKCAQEAzlIJfYIoQGXishIQGFNOcaVoeelqy7a731FJ+VfrqeR8WURQ6D+8 + 5hz7go+l3Z7IhTc/HbpGFJ5QJJNFSuSpLfZVyi+cKAUVheTivIniHFIRw37JbJ4+ + qWTlVe3uvOiZ0cA9S6LrbzqAUTLbH0JlWj36mvGIPICDr9YSEkIUKbenxjJlIpX8 + ECEBm8RU1aq3PUo/cVjmpqircynVJBbRCXZiHoxyLXNmh23d0fCPCabEYWhJhgaR + arkYRls5A14HGMI52F3ehnhED3k0mU8/lb4OzYgk34FjuZGmyRWIfrEKnqL4Uu2w + 3pmEvswG1WYG/3+YE80C5OpCE4BUKAzYSwIDAQAB + -----END RSA PUBLIC KEY----- + ''; + }; + }; + secure = true; + ssh.privkey.path = ; + ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAq5Ovdcsljr5dOl7+2sQNKpGpdX0SlOIuCZKEiWEp8g"; + }; iso = { cores = 1; managed = false; -- cgit v1.2.3 From e8148b62fdb39ed76361da5b7a6de88a8f800073 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 28 Jul 2017 12:04:37 +0200 Subject: l icarus: set correct ipv6 address --- krebs/3modules/lass/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'krebs/3modules') diff --git a/krebs/3modules/lass/default.nix b/krebs/3modules/lass/default.nix index 2e257b8a..c554391f 100644 --- a/krebs/3modules/lass/default.nix +++ b/krebs/3modules/lass/default.nix @@ -255,7 +255,7 @@ with import ; nets = rec { retiolum = { ip4.addr = "10.243.133.114"; - ip6.addr = "42:0000:0000:0000:0000:0000:d15f:1214"; + ip6.addr = "42:0:0:0:0:0:1ca0:1205"; aliases = [ "icarus.r" "cgit.icarus.r" -- cgit v1.2.3