diff options
Diffstat (limited to 'krebs/3modules/ci.nix')
| -rw-r--r-- | krebs/3modules/ci.nix | 222 |
1 files changed, 102 insertions, 120 deletions
diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix index 50db0b97..bb941a1f 100644 --- a/krebs/3modules/ci.nix +++ b/krebs/3modules/ci.nix @@ -39,148 +39,131 @@ let profileRoot = "/nix/var/nix/profiles/ci"; - imp = { - krebs.buildbot.master = { - slaves = { - testslave = "lasspass"; - }; + bcfg = config.services.buildbot-master; - change_source = mapAttrs' (name: repo: - nameValuePair name (concatMapStrings (url: '' - cs.append( - changes.GitPoller( - "${url}", - workdir='${name}-${elemAt(splitString "." url) 1}', branches=True, - project='${name}', - pollinterval=100 - ) + imp = { + services.buildbot-master = { + workers = [ "worker.Worker('testworker', 'pass')" ]; + + changeSource = mapAttrsToList (name: repo: + concatMapStringsSep "," (url: '' + changes.GitPoller( + "${url}", + workdir='${name}-${elemAt(splitString "." url) 1}', branches=True, + project='${name}', + pollinterval=100 ) - '') repo.urls) + '') repo.urls ) cfg.repos; - scheduler = mapAttrs' (name: repo: - nameValuePair name '' - sched.append( - schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter( - branch_re=".*", - project='${name}', - ), - treeStableTimer=60, - name="${name}-all-branches", - builderNames=[ - "${name}", - ] - ) - ) - sched.append( - schedulers.ForceScheduler( - name="${name}", - builderNames=[ - "${name}", - ] - ) - ) - '' - ) cfg.repos; - builder_pre = '' - from buildbot import interfaces - from buildbot.steps.shell import ShellCommand + schedulers = mapAttrsToList (name: repo: '' + schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter( + branch_re=".*", + project='${name}', + ), + treeStableTimer=60, + name="${name}-all-branches", + builderNames=[ + "${name}", + ] + ), + schedulers.ForceScheduler( + name="${name}", + builderNames=[ + "${name}", + ] + ) + '') cfg.repos; + + builders = []; - class StepToStartMoreSteps(ShellCommand): - def __init__(self, **kwargs): - ShellCommand.__init__(self, **kwargs) - - def addBuildSteps(self, steps_factories): - for sf in steps_factories: - step = interfaces.IBuildStepFactory(sf).buildStep() - step.setBuild(self.build) - step.setBuildSlave(self.build.slavebuilder.slave) - step_status = self.build.build_status.addStepWithName(step.name) - step.setStepStatus(step_status) - self.build.steps.append(step) - - def start(self): - props = self.build.getProperties() - new_steps = json.loads(props.getProperty('steps_json')) - for new_step in new_steps: - self.addBuildSteps([steps.ShellCommand( - name=str(new_step), - command=[ - "${pkgs.writeDash "build-stepper.sh" '' - set -xefu - profile=${shell.escape profileRoot}/$build_name - result=$("$build_script") - if [ -n "$result" ]; then - ${pkgs.nix}/bin/nix-env -p "$profile" --set "$result" - fi - ''}" - ], - env={ - "build_name": new_step, - "build_script": new_steps[new_step], - "NIX_REMOTE": "daemon", - "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src", - }, - timeout=90001, - workdir='build', # TODO figure out why we need this? - )]) - - ShellCommand.start(self) - - ''; + extraConfig = '' + # https://docs.buildbot.net/latest/manual/configuration/buildfactories.html + from buildbot.plugins import util, steps + from buildbot.process import buildstep, logobserver + from twisted.internet import defer + import json - builder = mapAttrs' (name: repo: - nameValuePair name '' - f_${name} = util.BuildFactory() - f_${name}.addStep(steps.Git( + class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): + def __init__(self, **kwargs): + kwargs = self.setupShellMixin(kwargs) + super().__init__(**kwargs) + self.observer = logobserver.BufferLogObserver() + self.addLogObserver('stdio', self.observer) + + def extract_stages(self, stdout): + stages = json.loads(stdout) + return stages + + @defer.inlineCallbacks + def run(self): + # run nix-instanstiate to generate the dict of stages + cmd = yield self.makeRemoteShellCommand() + yield self.runCommand(cmd) + + # if the command passes extract the list of stages + result = cmd.results() + if result == util.SUCCESS: + # create a ShellCommand for each stage and add them to the build + stages = self.extract_stages(self.observer.getStdout()) + self.build.addStepsAfterCurrentStep([ + steps.ShellCommand(name=stage, command=[stages[stage]]) + for stage in stages + ]) + + return result + + + ${concatStringsSep "\n" (mapAttrsToList (name: repo: '' + factory_${name} = util.BuildFactory() + factory_${name}.addStep(steps.Git( repourl=util.Property('repository', '${head repo.urls}'), method='clobber', mode='full', submodules=True, )) - f_${name}.addStep(steps.SetPropertyFromCommand( + factory_${name}.addStep(GenerateStagesCommand( env={ - "NIX_REMOTE": "daemon", - "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src", + "NIX_REMOTE": "daemon", + "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src", }, - name="get_steps", - command=["${getJobs}"], - extract_fn=lambda rc, stdout, stderr: { 'steps_json': stdout }, + name="Generate build stages", + command=[ + "${getJobs}" + ], + haltOnFailure=True, )) - f_${name}.addStep(StepToStartMoreSteps(command=["echo"])) # TODO remove dummy command from here - bu.append( + c['builders'].append( util.BuilderConfig( name="${name}", - slavenames=slavenames, - factory=f_${name} + workernames=['testworker'], + factory=factory_${name} ) ) - '' - ) cfg.repos; + '') cfg.repos)} + ''; enable = true; - web.enable = true; - irc = { - enable = true; - nick = "build|${hostname}"; - server = "irc.r"; - channels = [ "xxx" "noise" ]; - allowForce = true; - }; - extraConfig = '' - c['buildbotURL'] = "http://build.${hostname}.r/" - ''; + reporters = ['' + reporters.IRC( + host = "irc.r", + nick = "buildbot|${hostname}", + notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ], + channels = [{"channel": "#xxx"}], + ) + '']; + + buildbotUrl = "http://build.${hostname}.r/"; }; - krebs.buildbot.slave = { + services.buildbot-worker = { enable = true; - masterhost = "localhost"; - username = "testslave"; - password = "lasspass"; - packages = with pkgs; [ gnumake jq nix populate gnutar lzma gzip ]; + workerUser = "testworker"; + workerPass = "pass"; + packages = with pkgs; [ git gnutar gzip jq nix populate ]; }; system.activationScripts.buildbots-nix-profile = '' @@ -192,11 +175,10 @@ let users = { groups.buildbots.gid = genid "buildbots"; users = { - buildbotMaster.extraGroups = [ "buildbots" ]; - buildbotSlave.extraGroups = [ "buildbots" ]; + buildbot.extraGroups = [ "buildbots" ]; + bbworker.extraGroups = [ "buildbots" ]; }; }; }; in out - |