From 6c092cd20ca256b1cefa92aa779380cdb71c1313 Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 11 Nov 2016 01:34:18 +0100 Subject: buildbot: 0.8.4 -> 0.9.0rc2 --- krebs/3modules/buildbot/buildbot-worker.patch | 11 +++ krebs/3modules/buildbot/buildbot.patch | 11 +++ krebs/3modules/buildbot/master.nix | 95 +++++++++++++----------- krebs/3modules/buildbot/slave.nix | 101 +++++++++----------------- 4 files changed, 110 insertions(+), 108 deletions(-) create mode 100644 krebs/3modules/buildbot/buildbot-worker.patch create mode 100644 krebs/3modules/buildbot/buildbot.patch (limited to 'krebs') diff --git a/krebs/3modules/buildbot/buildbot-worker.patch b/krebs/3modules/buildbot/buildbot-worker.patch new file mode 100644 index 000000000..df6f7ed37 --- /dev/null +++ b/krebs/3modules/buildbot/buildbot-worker.patch @@ -0,0 +1,11 @@ +--- ./buildbot_worker/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100 ++++ ./buildbot_worker/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100 +@@ -76,7 +76,7 @@ + if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'): + tailBin = "/usr/xpg4/bin/tail" + else: +- tailBin = "/usr/bin/tail" ++ tailBin = "tail" + self.p = reactor.spawnProcess(self.pp, tailBin, + ("tail", "-f", "-n", "0", self.logfile), + env=os.environ, diff --git a/krebs/3modules/buildbot/buildbot.patch b/krebs/3modules/buildbot/buildbot.patch new file mode 100644 index 000000000..3a5794d82 --- /dev/null +++ b/krebs/3modules/buildbot/buildbot.patch @@ -0,0 +1,11 @@ +--- ./buildbot/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100 ++++ ./buildbot/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100 +@@ -76,7 +76,7 @@ + if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'): + tailBin = "/usr/xpg4/bin/tail" + else: +- tailBin = "/usr/bin/tail" ++ tailBin = "tail" + self.p = reactor.spawnProcess(self.pp, tailBin, + ("tail", "-f", "-n", "0", self.logfile), + env=os.environ, diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix index 9e144ee0e..b31661572 100644 --- a/krebs/3modules/buildbot/master.nix +++ b/krebs/3modules/buildbot/master.nix @@ -3,14 +3,10 @@ with import ; let - # https://github.com/NixOS/nixpkgs/issues/14026 - nixpkgs-fix = import (pkgs.fetchgit { - url = https://github.com/nixos/nixpkgs; - rev = "e026b5c243ea39810826e68362718f5d703fb5d0"; - sha256 = "87e0724910a6df0371f883f99a8cf42e366fb4119f676f6f74ffb404beca2632"; - }) {}; - - buildbot = nixpkgs-fix.buildbot; + buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-full (old:{ + patches = [ ./buildbot.patch ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ]; + }); buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' # -*- python -*- from buildbot.plugins import * @@ -18,11 +14,11 @@ let import json c = BuildmasterConfig = {} - c['slaves'] = [] - slaves = json.loads('${builtins.toJSON cfg.slaves}') - slavenames = [ s for s in slaves ] - for k,v in slaves.items(): - c['slaves'].append(buildslave.BuildSlave(k, v)) + c['workers'] = [] + workers = json.loads('${builtins.toJSON cfg.workers}') + workernames = [ s for s in workers ] + for k,v in workers.items(): + c['workers'].append(worker.Worker(k, v)) # TODO: configure protocols? c['protocols'] = {'pb': {'port': 9989}} @@ -63,32 +59,46 @@ let ####### Status - c['status'] = st = [] + c['services'] = [] # If you want to configure this url, override with extraConfig c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/" ${optionalString (cfg.web.enable) '' - from buildbot.status import html - from buildbot.status.web import authz, auth - authz_cfg=authz.Authz( - auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]), - # TODO: configure harder - gracefulShutdown = False, - forceBuild = 'auth', - forceAllBuilds = 'auth', - pingBuilder = False, - stopBuild = 'auth', - stopAllBuilds = 'auth', - cancelPendingBuild = 'auth' - ) + from buildbot.plugins import util + + #authz_cfg=authz.Authz( + # auth=auth.BasicAuth([ ]), + # # TODO: configure harder + # gracefulShutdown = False, + # forceBuild = 'auth', + # forceAllBuilds = 'auth', + # pingBuilder = False, + # stopBuild = 'auth', + # stopAllBuilds = 'auth', + # cancelPendingBuild = 'auth' + #) # TODO: configure krebs.nginx - st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg)) + c['www'] = dict( + port = ${toString cfg.web.port}, + plugins = { 'waterfall_view':{}, 'console_view':{} } + ) + c['www']['auth'] = util.UserPasswordAuth({"${cfg.web.username}":"${cfg.web.password}"}) + c['www']['authz'] = util.Authz( + allowRules = [ + util.StopBuildEndpointMatcher(role="admins"), + util.ForceBuildEndpointMatcher(role="admins"), + util.RebuildBuildEndpointMatcher(role="admins") + ], + roleMatchers = [ + util.RolesFromEmails(admins=["${cfg.web.username}"]) + ] + ) ''} ${optionalString (cfg.irc.enable) '' - from buildbot.status import words - irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}", + from buildbot.plugins import reporters + irc = reporters.IRC("${cfg.irc.server}", "${cfg.irc.nick}", channels=${builtins.toJSON cfg.irc.channels}, notify_events={ 'success': 1, @@ -97,7 +107,7 @@ let 'successToFailure': 1, 'failureToSuccess': 1, }${optionalString cfg.irc.allowForce ",allowForce=True"}) - c['status'].append(irc) + c['services'].append(irc) ''} ${ concatStringsSep "\n" @@ -150,12 +160,12 @@ let ''; }; - slaves = mkOption { + workers = mkOption { default = {}; type = types.attrsOf types.str; description = '' - Attrset of slavenames with their passwords - slavename = slavepassword + Attrset of workernames with their passwords + workername = workerpassword ''; }; @@ -283,8 +293,12 @@ let options = { enable = mkEnableOption "Buildbot Master IRC Status"; channels = mkOption { - default = [ "nix-buildbot-meetup" ]; - type = with types; listOf str; + default = [ { channel = "nix-buildbot-meetup";} ]; + example = literalExample ''[ + {channel = "nix-buildbot-meetup";} + {channel = "nix-buildbot-lol"; "password" = "lol";} + ]''; + type = with types; listOf (attrsOf str); description = '' irc channels the bot should connect to ''; @@ -333,7 +347,7 @@ let }; users.extraGroups.buildbotMaster = { - gid = 672626386; + gid = genid "buildbotMaster"; }; systemd.services.buildbotMaster = { @@ -350,8 +364,6 @@ let secretsdir = shell.escape (toString ); in { PermissionsStartOnly = true; - Type = "forking"; - PIDFile = "${workdir}/twistd.pid"; # TODO: maybe also prepare buildbot.tac? ExecStartPre = pkgs.writeDash "buildbot-master-init" '' set -efux @@ -375,9 +387,8 @@ let chmod 700 -R ${workdir} chown buildbotMaster:buildbotMaster -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; - ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; - ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + ExecStart = "${buildbot}/bin/buildbot start --nodaemon ${workdir}"; + # ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; PrivateTmp = "true"; User = "buildbotMaster"; Restart = "always"; diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 650594a6c..95b547081 100644 --- a/krebs/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -2,53 +2,21 @@ with import ; let - buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' - import os - - from buildslave.bot import BuildSlave - from twisted.application import service - - basedir = '${cfg.workDir}' - rotateLength = 10000000 - maxRotatedFiles = 10 - - application = service.Application('buildslave') - - from twisted.python.logfile import LogFile - from twisted.python.log import ILogObserver, FileLogObserver - logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, - maxRotatedFiles=maxRotatedFiles) - application.setComponent(ILogObserver, FileLogObserver(logfile).emit) - - buildmaster_host = '${cfg.masterhost}' - # TODO: masterport? - port = 9989 - slavename = '${cfg.username}' - passwd = '${cfg.password}' - keepalive = 600 - usepty = 0 - umask = None - maxdelay = 300 - allow_shutdown = None - - ${cfg.extraConfig} - - s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, - keepalive, usepty, umask=umask, maxdelay=maxdelay, - allow_shutdown=allow_shutdown) - s.setServiceParent(application) - ''; default-packages = [ pkgs.git pkgs.bash ]; - cfg = config.krebs.buildbot.slave; + buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-worker (old:{ + patches = [ ./buildbot-worker.patch ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ]; + }); + cfg = config.krebs.buildbot.worker; api = { - enable = mkEnableOption "Buildbot Slave"; + enable = mkEnableOption "Buildbot worker"; workDir = mkOption { - default = "/var/lib/buildbot/slave"; + default = "/var/lib/buildbot/worker"; type = types.str; description = '' - Path to build bot slave directory. + Path to build bot worker directory. Will be created on startup. ''; }; @@ -64,30 +32,30 @@ let username = mkOption { type = types.str; description = '' - slavename used to authenticate with master + workername used to authenticate with master ''; }; password = mkOption { type = types.str; description = '' - slave password used to authenticate with master + worker password used to authenticate with master ''; }; contact = mkOption { - default = "nix slave "; + default = "nix worker "; type = types.str; description = '' - contact to be announced by buildslave + contact to be announced by buildworker ''; }; description = mkOption { - default = "Nix Generated BuildSlave"; + default = "Nix Generated Buildworker"; type = types.str; description = '' - description for hostto be announced by buildslave + description for hostto be announced by buildworker ''; }; @@ -95,7 +63,7 @@ let default = [ pkgs.git ]; type = with types; listOf package; description = '' - packages which should be in path for buildslave + packages which should be in path for buildworker ''; }; @@ -106,7 +74,7 @@ let }; type = types.attrsOf types.str; description = '' - extra environment variables to be provided to the buildslave service + extra environment variables to be provided to the buildworker service if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here. ''; }; @@ -119,26 +87,26 @@ let keepalive = 600 ''; description = '' - extra config evaluated before calling BuildSlave init in .tac file + extra config evaluated before calling Buildworker init in .tac file ''; }; }; imp = { - users.extraUsers.buildbotSlave = { - uid = genid "buildbotSlave"; - description = "Buildbot Slave"; + users.extraUsers.buildbotworker = { + uid = genid "buildbotworker"; + description = "Buildbot worker"; home = cfg.workDir; createHome = false; }; - users.extraGroups.buildbotSlave = { - gid = 1408105834; + users.extraGroups.buildbotworker = { + gid = genid "buildbotworker"; }; - systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = { - description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}"; + systemd.services."buildbotworker-${cfg.username}-${cfg.masterhost}" = { + description = "Buildbot worker for ${cfg.username}@${cfg.masterhost}"; after = [ "network.target" ]; wantedBy = [ "multi-user.target" ]; path = default-packages ++ cfg.packages; @@ -152,27 +120,28 @@ let workdir = shell.escape cfg.workDir; contact = shell.escape cfg.contact; description = shell.escape cfg.description; - buildbot = pkgs.buildbot-slave; - # TODO:make this + masterhost = shell.escape cfg.masterhost; + username = shell.escape cfg.username; + password = shell.escape cfg.password; in { PermissionsStartOnly = true; Type = "forking"; PIDFile = "${workdir}/twistd.pid"; - # TODO: maybe also prepare buildbot.tac? - ExecStartPre = pkgs.writeDash "buildbot-master-init" '' + ExecStartPre = pkgs.writeDash "buildbot-slave-init" '' set -efux mkdir -p ${workdir}/info - cp ${buildbot-slave-init} ${workdir}/buildbot.tac + # TODO: cleanup .tac file? + ${buildbot}/bin/buildbot-worker create-worker ${workdir} ${masterhost} ${username} ${password} echo ${contact} > ${workdir}/info/admin echo ${description} > ${workdir}/info/host - chown buildbotSlave:buildbotSlave -R ${workdir} + chown buildbotworker:buildbotworker -R ${workdir} chmod 700 -R ${workdir} ''; - ExecStart = "${buildbot}/bin/buildslave start ${workdir}"; - ExecStop = "${buildbot}/bin/buildslave stop ${workdir}"; + ExecStart = "${buildbot}/bin/buildbot-worker start ${workdir}"; + ExecStop = "${buildbot}/bin/buildbot-worker stop ${workdir}"; PrivateTmp = "true"; - User = "buildbotSlave"; + User = "buildbotworker"; Restart = "always"; RestartSec = "10"; }; @@ -180,6 +149,6 @@ let }; in { - options.krebs.buildbot.slave = api; + options.krebs.buildbot.worker = api; config = lib.mkIf cfg.enable imp; } -- cgit v1.2.3