diff options
author | lassulus <lass@aidsballs.de> | 2015-12-16 15:46:23 +0100 |
---|---|---|
committer | lassulus <lass@aidsballs.de> | 2015-12-16 15:46:23 +0100 |
commit | 07f4510ad0bff0f9d72e5dcee279a30a0d5f8da3 (patch) | |
tree | 5848f7727500895df71a523a3405087448ba9e4a /makefu/3modules/buildbot | |
parent | d94784efbecbd437ec6268bb9bfca57a8088d6e8 (diff) | |
parent | 8f18b00ab141df92b7df4725a18bb3283b184d76 (diff) |
Merge remote-tracking branch 'gum/master'
Diffstat (limited to 'makefu/3modules/buildbot')
-rw-r--r-- | makefu/3modules/buildbot/master.nix | 226 | ||||
-rw-r--r-- | makefu/3modules/buildbot/slave.nix | 159 |
2 files changed, 385 insertions, 0 deletions
diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix new file mode 100644 index 00000000..5d340f89 --- /dev/null +++ b/makefu/3modules/buildbot/master.nix @@ -0,0 +1,226 @@ +{ config, pkgs, lib, ... }: + +with lib; +let + buildbot = pkgs.buildbot; + buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' + # -*- python -*- + from buildbot.plugins import * + + c = BuildmasterConfig = {} + + c['slaves'] = [] + # TODO: template potential buildslaves + # TODO: set password? + slavenames= [ 'testslave' ] + for i in slavenames: + c['slaves'].append(buildslave.BuildSlave(i, "krebspass")) + + c['protocols'] = {'pb': {'port': 9989}} + + ####### Build Inputs + stockholm_repo = 'http://cgit.gum/stockholm' + c['change_source'] = [] + c['change_source'].append(changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=300)) + + ####### Build Scheduler + # TODO: configure scheduler + important_files = util.ChangeFilter( + project_re="^((krebs|share)/.*|Makefile|default.nix)", + branch='master') + c['schedulers'] = [] + c['schedulers'].append(schedulers.SingleBranchScheduler( + name="all-important-files", + change_filter=important_files, + # 3 minutes stable tree + treeStableTimer=3*60, + builderNames=["runtests"])) + c['schedulers'].append(schedulers.ForceScheduler( + name="force", + builderNames=["runtests"])) + ###### The actual build + factory = util.BuildFactory() + factory.addStep(steps.Git(repourl=stockholm_repo, mode='incremental')) + + deps = [ "gnumake", "jq" ] + factory.addStep(steps.ShellCommand(command=["nix-shell", "-p" ] + deps )) + factory.addStep(steps.ShellCommand(env={"LOGNAME": "shared"}, + command=["make", "get=krebs.deploy", + "system=test-centos7"])) + + # TODO: different Builders? + c['builders'] = [] + c['builders'].append( + util.BuilderConfig(name="runtests", + # TODO: only some slaves being used in builder? + slavenames=slavenames, + factory=factory)) + + ####### Status of Builds + c['status'] = [] + + from buildbot.status import html + from buildbot.status.web import authz, auth + # TODO: configure if http is wanted + authz_cfg=authz.Authz( + # TODO: configure user/pw + auth=auth.BasicAuth([("krebs","bob")]), + gracefulShutdown = False, + forceBuild = 'auth', + forceAllBuilds = 'auth', + pingBuilder = False, + stopBuild = False, + stopAllBuilds = False, + cancelPendingBuild = False, + ) + # TODO: configure nginx + c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg)) + + from buildbot.status import words + ${optionalString (cfg.irc.enable) '' + irc = words.IRC("${cfg.irc.server}", "krebsbuild", + # TODO: multiple channels + channels=["${cfg.irc.channel}"], + notify_events={ + 'success': 1, + 'failure': 1, + 'exception': 1, + 'successToFailure': 1, + 'failureToSuccess': 1, + }${optionalString cfg.irc.allowForce ",allowForce=True"}) + c['status'].append(irc) + ''} + + ####### PROJECT IDENTITY + c['title'] = "Stockholm" + c['titleURL'] = "http://krebsco.de" + + c['buildbotURL'] = "http://buildbot.krebsco.de/" + + ####### DB URL + c['db'] = { + 'db_url' : "sqlite:///state.sqlite", + } + ${cfg.extraConfig} + ''; + + cfg = config.makefu.buildbot.master; + + api = { + enable = mkEnableOption "Buildbot Master"; + + workDir = mkOption { + default = "/var/lib/buildbot/master"; + type = types.str; + description = '' + Path to build bot master directory. + Will be created on startup. + ''; + }; + irc = mkOption { + default = {}; + type = types.submodule ({ config, ... }: { + options = { + enable = mkEnableOption "Buildbot Master IRC Status"; + channel = mkOption { + default = "nix-buildbot-meetup"; + type = types.str; + description = '' + irc channel the bot should connect to + ''; + }; + allowForce = mkOption { + default = false; + type = types.bool; + description = '' + Determines if builds can be forced via IRC + ''; + }; + nick = mkOption { + default = "nix-buildbot"; + type = types.str; + description = '' + nickname for IRC + ''; + }; + server = mkOption { + default = "irc.freenode.net"; + type = types.str; + description = '' + Buildbot Status IRC Server to connect to + ''; + }; + }; + }); + }; + extraConfig = mkOption { + default = ""; + type = types.lines; + description = '' + extra config appended to the generated master.cfg + ''; + }; + }; + + imp = { + + users.extraUsers.buildbotMaster = { + uid = 672626386; #genid buildbotMaster + description = "Buildbot Master"; + home = cfg.workDir; + createHome = false; + }; + + users.extraGroups.buildbotMaster = { + gid = 672626386; + }; + + systemd.services.buildbotMaster = { + description = "Buildbot Master"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = let + workdir="${lib.shell.escape cfg.workDir}"; + in { + PermissionsStartOnly = true; + Type = "forking"; + PIDFile = "${workdir}/twistd.pid"; + # TODO: maybe also prepare buildbot.tac? + ExecStartPre = pkgs.writeScript "buildbot-master-init" '' + #!/bin/sh + set -efux + if [ ! -e ${workdir} ];then + mkdir -p ${workdir} + ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir} + fi + # always override the master.cfg + cp ${buildbot-master-config} ${workdir}/master.cfg + # sanity + ${buildbot}/bin/buildbot checkconfig ${workdir} + + # TODO: maybe upgrade? not sure about this + # normally we should write buildbot.tac by our own + # ${buildbot}/bin/buildbot upgrade-master ${workdir} + + chmod 700 -R ${workdir} + chown buildbotMaster:buildbotMaster -R ${workdir} + ''; + ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; + ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; + ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + PrivateTmp = "true"; + User = "buildbotMaster"; + Restart = "always"; + RestartSec = "10"; + }; + }; + }; +in +{ + options.makefu.buildbot.master = api; + config = mkIf cfg.enable imp; +} diff --git a/makefu/3modules/buildbot/slave.nix b/makefu/3modules/buildbot/slave.nix new file mode 100644 index 00000000..188a9283 --- /dev/null +++ b/makefu/3modules/buildbot/slave.nix @@ -0,0 +1,159 @@ +{ config, pkgs, lib, ... }: + +with lib; +let + buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" '' + import os + + from buildslave.bot import BuildSlave + from twisted.application import service + + basedir = '${cfg.workDir}' + rotateLength = 10000000 + maxRotatedFiles = 10 + + application = service.Application('buildslave') + + from twisted.python.logfile import LogFile + from twisted.python.log import ILogObserver, FileLogObserver + logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, + maxRotatedFiles=maxRotatedFiles) + application.setComponent(ILogObserver, FileLogObserver(logfile).emit) + + buildmaster_host = '${cfg.masterhost}' + # TODO: masterport? + port = 9989 + slavename = '${cfg.username}' + passwd = '${cfg.password}' + keepalive = 600 + usepty = 0 + umask = None + maxdelay = 300 + allow_shutdown = None + + ${cfg.extraConfig} + + s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, + keepalive, usepty, umask=umask, maxdelay=maxdelay, + allow_shutdown=allow_shutdown) + s.setServiceParent(application) + ''; + + cfg = config.makefu.buildbot.slave; + + api = { + enable = mkEnableOption "Buildbot Slave"; + + workDir = mkOption { + default = "/var/lib/buildbot/slave"; + type = types.str; + description = '' + Path to build bot slave directory. + Will be created on startup. + ''; + }; + + masterhost = mkOption { + default = "localhost"; + type = types.str; + description = '' + Hostname/IP of the buildbot master + ''; + }; + + username = mkOption { + type = types.str; + description = '' + slavename used to authenticate with master + ''; + }; + + password = mkOption { + type = types.str; + description = '' + slave password used to authenticate with master + ''; + }; + + contact = mkOption { + default = "nix slave <buildslave@${config.networking.hostName}>"; + type = types.str; + description = '' + contact to be announced by buildslave + ''; + }; + + description = mkOption { + default = "Nix Generated BuildSlave"; + type = types.str; + description = '' + description for hostto be announced by buildslave + ''; + }; + + extraConfig = mkOption { + default = ""; + type = types.lines; + example = '' + port = 443 + keepalive = 600 + ''; + description = '' + extra config evaluated before calling BuildSlave init in .tac file + ''; + }; + }; + + imp = { + + users.extraUsers.buildbotSlave = { + uid = 1408105834; #genid buildbotMaster + description = "Buildbot Slave"; + home = cfg.workDir; + createHome = false; + }; + + users.extraGroups.buildbotSlave = { + gid = 1408105834; + }; + + systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = { + description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = let + workdir = "${lib.shell.escape cfg.workDir}"; + contact = "${lib.shell.escape cfg.contact}"; + description = "${lib.shell.escape cfg.description}"; + buildbot = pkgs.buildbot-slave; + # TODO:make this + in { + PermissionsStartOnly = true; + Type = "forking"; + PIDFile = "${workdir}/twistd.pid"; + # TODO: maybe also prepare buildbot.tac? + ExecStartPre = pkgs.writeScript "buildbot-master-init" '' + #!/bin/sh + set -efux + mkdir -p ${workdir}/info + cp ${buildbot-slave-init} ${workdir}/buildbot.tac + echo ${contact} > ${workdir}/info/admin + echo ${description} > ${workdir}/info/host + + chown buildbotSlave:buildbotSlave -R ${workdir} + chmod 700 -R ${workdir} + ''; + ExecStart = "${buildbot}/bin/buildslave start ${workdir}"; + ExecStop = "${buildbot}/bin/buildslave stop ${workdir}"; + PrivateTmp = "true"; + User = "buildbotSlave"; + Restart = "always"; + RestartSec = "10"; + }; + }; + }; +in +{ + options.makefu.buildbot.slave = api; + config = mkIf cfg.enable imp; +} |