summaryrefslogtreecommitdiffstats
path: root/krebs
diff options
context:
space:
mode:
Diffstat (limited to 'krebs')
-rw-r--r--krebs/2configs/shared-buildbot.nix23
-rw-r--r--krebs/3modules/buildbot/buildbot-worker.patch11
-rw-r--r--krebs/3modules/buildbot/buildbot.patch11
-rw-r--r--krebs/3modules/buildbot/master.nix94
-rw-r--r--krebs/3modules/buildbot/slave.nix107
-rw-r--r--krebs/5pkgs/simple/buildbot/default.nix82
-rw-r--r--krebs/5pkgs/simple/buildbot/worker.nix24
7 files changed, 126 insertions, 226 deletions
diff --git a/krebs/2configs/shared-buildbot.nix b/krebs/2configs/shared-buildbot.nix
index a9e5afc75..b534f0b62 100644
--- a/krebs/2configs/shared-buildbot.nix
+++ b/krebs/2configs/shared-buildbot.nix
@@ -26,11 +26,12 @@ in {
nix.gc.automatic = true;
nix.gc.dates = "05:23";
networking.firewall.allowedTCPPorts = [ 80 8010 9989 ];
+
krebs.buildbot.master = let
stockholm-mirror-url = "http://cgit.${hostname}.r/stockholm" ;
in {
- workers = {
- testworker = "krebspass";
+ slaves = {
+ testslave = "krebspass";
};
change_source.stockholm = ''
stockholm_repo = '${stockholm-mirror-url}'
@@ -119,7 +120,7 @@ in {
system={}".format(i)])
bu.append(util.BuilderConfig(name="fast-tests",
- workernames=workernames,
+ slavenames=slavenames,
factory=f))
'';
@@ -131,25 +132,25 @@ in {
bu.append(util.BuilderConfig(name="build-local",
- workernames=workernames,
+ slavenames=slavenames,
factory=f))
'';
# slow-tests = ''
# s = util.BuildFactory()
# s.addStep(grab_repo)
#
-# # worker needs 2 files:
+# # slave needs 2 files:
# # * cac.json
# # * retiolum
-# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", workerdest="cac.json"))
-# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", workerdest="retiolum.rsa_key.priv"))
+# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json"))
+# s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv"))
# addShell(s, name="infest-cac-centos7",env=env,
# sigtermTime=60, # SIGTERM 1 minute before SIGKILL
# timeout=10800, # 3h
# command=nixshell + ["infest-cac-centos7"])
#
# bu.append(util.BuilderConfig(name="full-tests",
-# workernames=workernames,
+# slavenames=slavenames,
# factory=s))
# '';
};
@@ -161,7 +162,7 @@ in {
enable = true;
nick = "${hostname}bot";
server = "ni.r";
- channels = [ { channel = "retiolum"; } ];
+ channels = [ "retiolum" ];
allowForce = true;
};
extraConfig = ''
@@ -169,10 +170,10 @@ in {
'';
};
- krebs.buildbot.worker = {
+ krebs.buildbot.slave = {
enable = true;
masterhost = "localhost";
- username = "testworker";
+ username = "testslave";
password = "krebspass";
packages = with pkgs; [ gnumake jq nix populate ];
# all nix commands will need a working nixpkgs installation
diff --git a/krebs/3modules/buildbot/buildbot-worker.patch b/krebs/3modules/buildbot/buildbot-worker.patch
deleted file mode 100644
index df6f7ed37..000000000
--- a/krebs/3modules/buildbot/buildbot-worker.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- ./buildbot_worker/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100
-+++ ./buildbot_worker/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100
-@@ -76,7 +76,7 @@
- if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'):
- tailBin = "/usr/xpg4/bin/tail"
- else:
-- tailBin = "/usr/bin/tail"
-+ tailBin = "tail"
- self.p = reactor.spawnProcess(self.pp, tailBin,
- ("tail", "-f", "-n", "0", self.logfile),
- env=os.environ,
diff --git a/krebs/3modules/buildbot/buildbot.patch b/krebs/3modules/buildbot/buildbot.patch
deleted file mode 100644
index 3a5794d82..000000000
--- a/krebs/3modules/buildbot/buildbot.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- ./buildbot/scripts/logwatcher.py 2016-11-10 23:25:46.956000000 +0100
-+++ ./buildbot/scripts/logwatcher.py.fix 2016-11-10 23:24:33.225000000 +0100
-@@ -76,7 +76,7 @@
- if platform.system().lower() == 'sunos' and os.path.exists('/usr/xpg4/bin/tail'):
- tailBin = "/usr/xpg4/bin/tail"
- else:
-- tailBin = "/usr/bin/tail"
-+ tailBin = "tail"
- self.p = reactor.spawnProcess(self.pp, tailBin,
- ("tail", "-f", "-n", "0", self.logfile),
- env=os.environ,
diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
index d75e6c880..6c7af6da5 100644
--- a/krebs/3modules/buildbot/master.nix
+++ b/krebs/3modules/buildbot/master.nix
@@ -3,10 +3,14 @@
with import <stockholm/lib>;
let
- buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-full (old:{
- patches = [ ./buildbot.patch ];
- propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ];
- });
+ # https://github.com/NixOS/nixpkgs/issues/14026
+ nixpkgs-fix = import (pkgs.fetchgit {
+ url = https://github.com/nixos/nixpkgs;
+ rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
+ sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8";
+ }) {};
+
+ buildbot = nixpkgs-fix.buildbot;
buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
# -*- python -*-
from buildbot.plugins import *
@@ -14,11 +18,11 @@ let
import json
c = BuildmasterConfig = {}
- c['workers'] = []
- workers = json.loads('${builtins.toJSON cfg.workers}')
- workernames = [ s for s in workers ]
- for k,v in workers.items():
- c['workers'].append(worker.Worker(k, v))
+ c['slaves'] = []
+ slaves = json.loads('${builtins.toJSON cfg.slaves}')
+ slavenames = [ s for s in slaves ]
+ for k,v in slaves.items():
+ c['slaves'].append(buildslave.BuildSlave(k, v))
# TODO: configure protocols?
c['protocols'] = {'pb': {'port': 9989}}
@@ -59,45 +63,32 @@ let
####### Status
- c['services'] = []
+ c['status'] = st = []
# If you want to configure this url, override with extraConfig
c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
${optionalString (cfg.web.enable) ''
- from buildbot.plugins import util
-
- #authz_cfg=authz.Authz(
- # auth=auth.BasicAuth([ ]),
- # # TODO: configure harder
- # gracefulShutdown = False,
- # forceBuild = 'auth',
- # forceAllBuilds = 'auth',
- # pingBuilder = False,
- # stopBuild = 'auth',
- # stopAllBuilds = 'auth',
- # cancelPendingBuild = 'auth'
- #)
- c['www'] = dict(
- port = ${toString cfg.web.port},
- plugins = { 'waterfall_view':{}, 'console_view':{} }
- )
- c['www']['auth'] = util.UserPasswordAuth({"${cfg.web.username}":"${cfg.web.password}"})
- c['www']['authz'] = util.Authz(
- allowRules = [
- util.StopBuildEndpointMatcher(role="admins"),
- util.ForceBuildEndpointMatcher(role="admins"),
- util.RebuildBuildEndpointMatcher(role="admins")
- ],
- roleMatchers = [
- util.RolesFromEmails(admins=["${cfg.web.username}"])
- ]
+ from buildbot.status import html
+ from buildbot.status.web import authz, auth
+ authz_cfg=authz.Authz(
+ auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
+ # TODO: configure harder
+ gracefulShutdown = False,
+ forceBuild = 'auth',
+ forceAllBuilds = 'auth',
+ pingBuilder = False,
+ stopBuild = 'auth',
+ stopAllBuilds = 'auth',
+ cancelPendingBuild = 'auth'
)
+ # TODO: configure krebs.nginx
+ st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
''}
${optionalString (cfg.irc.enable) ''
- from buildbot.plugins import reporters
- irc = reporters.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+ from buildbot.status import words
+ irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
channels=${builtins.toJSON cfg.irc.channels},
notify_events={
'success': 1,
@@ -106,7 +97,7 @@ let
'successToFailure': 1,
'failureToSuccess': 1,
}${optionalString cfg.irc.allowForce ",allowForce=True"})
- c['services'].append(irc)
+ c['status'].append(irc)
''}
${ concatStringsSep "\n"
@@ -159,12 +150,12 @@ let
'';
};
- workers = mkOption {
+ slaves = mkOption {
default = {};
type = types.attrsOf types.str;
description = ''
- Attrset of workernames with their passwords
- workername = workerpassword
+ Attrset of slavenames with their passwords
+ slavename = slavepassword
'';
};
@@ -292,12 +283,8 @@ let
options = {
enable = mkEnableOption "Buildbot Master IRC Status";
channels = mkOption {
- default = [ { channel = "nix-buildbot-meetup";} ];
- example = literalExample ''[
- {channel = "nix-buildbot-meetup";}
- {channel = "nix-buildbot-lol"; "password" = "lol";}
- ]'';
- type = with types; listOf (attrsOf str);
+ default = [ "nix-buildbot-meetup" ];
+ type = with types; listOf str;
description = ''
irc channels the bot should connect to
'';
@@ -346,7 +333,7 @@ let
};
users.extraGroups.buildbotMaster = {
- gid = genid "buildbotMaster";
+ gid = 672626386;
};
systemd.services.buildbotMaster = {
@@ -363,6 +350,8 @@ let
secretsdir = shell.escape (toString <secrets>);
in {
PermissionsStartOnly = true;
+ Type = "forking";
+ PIDFile = "${workdir}/twistd.pid";
# TODO: maybe also prepare buildbot.tac?
ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
set -efux
@@ -386,8 +375,9 @@ let
chmod 700 -R ${workdir}
chown buildbotMaster:buildbotMaster -R ${workdir}
'';
- ExecStart = "${buildbot}/bin/buildbot start --nodaemon ${workdir}";
- # ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
+ ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+ ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+ ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
PrivateTmp = "true";
User = "buildbotMaster";
Restart = "always";
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
index 95b547081..932923ae5 100644
--- a/krebs/3modules/buildbot/slave.nix
+++ b/krebs/3modules/buildbot/slave.nix
@@ -2,21 +2,59 @@
with import <stockholm/lib>;
let
+ nixpkgs-fix = import (pkgs.fetchgit {
+ url = https://github.com/nixos/nixpkgs;
+ rev = "e026b5c243ea39810826e68362718f5d703fb5d0";
+ sha256 = "11lqd480bi6xbi7xbh4krrxmbp6a6iafv1d0q3sj461al0x0has8";
+ }) {};
+
+ buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
+ import os
+
+ from buildslave.bot import BuildSlave
+ from twisted.application import service
+
+ basedir = '${cfg.workDir}'
+ rotateLength = 10000000
+ maxRotatedFiles = 10
+
+ application = service.Application('buildslave')
+
+ from twisted.python.logfile import LogFile
+ from twisted.python.log import ILogObserver, FileLogObserver
+ logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+ maxRotatedFiles=maxRotatedFiles)
+ application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+ buildmaster_host = '${cfg.masterhost}'
+ # TODO: masterport?
+ port = 9989
+ slavename = '${cfg.username}'
+ passwd = '${cfg.password}'
+ keepalive = 600
+ usepty = 0
+ umask = None
+ maxdelay = 300
+ allow_shutdown = None
+
+ ${cfg.extraConfig}
+
+ s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
+ keepalive, usepty, umask=umask, maxdelay=maxdelay,
+ allow_shutdown=allow_shutdown)
+ s.setServiceParent(application)
+ '';
default-packages = [ pkgs.git pkgs.bash ];
- buildbot = pkgs.stdenv.lib.overrideDerivation pkgs.buildbot-worker (old:{
- patches = [ ./buildbot-worker.patch ];
- propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.coreutils ];
- });
- cfg = config.krebs.buildbot.worker;
+ cfg = config.krebs.buildbot.slave;
api = {
- enable = mkEnableOption "Buildbot worker";
+ enable = mkEnableOption "Buildbot Slave";
workDir = mkOption {
- default = "/var/lib/buildbot/worker";
+ default = "/var/lib/buildbot/slave";
type = types.str;
description = ''
- Path to build bot worker directory.
+ Path to build bot slave directory.
Will be created on startup.
'';
};
@@ -32,30 +70,30 @@ let
username = mkOption {
type = types.str;
description = ''
- workername used to authenticate with master
+ slavename used to authenticate with master
'';
};
password = mkOption {
type = types.str;
description = ''
- worker password used to authenticate with master
+ slave password used to authenticate with master
'';
};
contact = mkOption {
- default = "nix worker <buildworker@${config.networking.hostName}>";
+ default = "nix slave <buildslave@${config.networking.hostName}>";
type = types.str;
description = ''
- contact to be announced by buildworker
+ contact to be announced by buildslave
'';
};
description = mkOption {
- default = "Nix Generated Buildworker";
+ default = "Nix Generated BuildSlave";
type = types.str;
description = ''
- description for hostto be announced by buildworker
+ description for hostto be announced by buildslave
'';
};
@@ -63,7 +101,7 @@ let
default = [ pkgs.git ];
type = with types; listOf package;
description = ''
- packages which should be in path for buildworker
+ packages which should be in path for buildslave
'';
};
@@ -74,7 +112,7 @@ let
};
type = types.attrsOf types.str;
description = ''
- extra environment variables to be provided to the buildworker service
+ extra environment variables to be provided to the buildslave service
if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
'';
};
@@ -87,26 +125,26 @@ let
keepalive = 600
'';
description = ''
- extra config evaluated before calling Buildworker init in .tac file
+ extra config evaluated before calling BuildSlave init in .tac file
'';
};
};
imp = {
- users.extraUsers.buildbotworker = {
- uid = genid "buildbotworker";
- description = "Buildbot worker";
+ users.extraUsers.buildbotSlave = {
+ uid = genid "buildbotSlave";
+ description = "Buildbot Slave";
home = cfg.workDir;
createHome = false;
};
- users.extraGroups.buildbotworker = {
- gid = genid "buildbotworker";
+ users.extraGroups.buildbotSlave = {
+ gid = 1408105834;
};
- systemd.services."buildbotworker-${cfg.username}-${cfg.masterhost}" = {
- description = "Buildbot worker for ${cfg.username}@${cfg.masterhost}";
+ systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
+ description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
after = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
path = default-packages ++ cfg.packages;
@@ -120,28 +158,27 @@ let
workdir = shell.escape cfg.workDir;
contact = shell.escape cfg.contact;
description = shell.escape cfg.description;
- masterhost = shell.escape cfg.masterhost;
- username = shell.escape cfg.username;
- password = shell.escape cfg.password;
+ buildbot = nixpkgs-fix.buildbot-slave;
+ # TODO:make this
in {
PermissionsStartOnly = true;
Type = "forking";
PIDFile = "${workdir}/twistd.pid";
- ExecStartPre = pkgs.writeDash "buildbot-slave-init" ''
+ # TODO: maybe also prepare buildbot.tac?
+ ExecStartPre = pkgs.writeDash "buildbot-master-init" ''
set -efux
mkdir -p ${workdir}/info
- # TODO: cleanup .tac file?
- ${buildbot}/bin/buildbot-worker create-worker ${workdir} ${masterhost} ${username} ${password}
+ cp ${buildbot-slave-init} ${workdir}/buildbot.tac
echo ${contact} > ${workdir}/info/admin
echo ${description} > ${workdir}/info/host
- chown buildbotworker:buildbotworker -R ${workdir}
+ chown buildbotSlave:buildbotSlave -R ${workdir}
chmod 700 -R ${workdir}
'';
- ExecStart = "${buildbot}/bin/buildbot-worker start ${workdir}";
- ExecStop = "${buildbot}/bin/buildbot-worker stop ${workdir}";
+ ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
+ ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
PrivateTmp = "true";
- User = "buildbotworker";
+ User = "buildbotSlave";
Restart = "always";
RestartSec = "10";
};
@@ -149,6 +186,6 @@ let
};
in
{
- options.krebs.buildbot.worker = api;
+ options.krebs.buildbot.slave = api;
config = lib.mkIf cfg.enable imp;
}
diff --git a/krebs/5pkgs/simple/buildbot/default.nix b/krebs/5pkgs/simple/buildbot/default.nix
deleted file mode 100644
index 37eea5fd9..000000000
--- a/krebs/5pkgs/simple/buildbot/default.nix
+++ /dev/null
@@ -1,82 +0,0 @@
-{ pkgs, stdenv, pythonPackages, fetchurl, coreutils, plugins ? [] }:
-
-pythonPackages.buildPythonApplication (rec {
- name = "${pname}-${version}";
- pname = "buildbot";
- version = "0.9.4";
- src = fetchurl {
- url = "mirror://pypi/b/${pname}/${name}.tar.gz";
- sha256 = "0wklrn4fszac9wi8zw3vbsznwyff6y57cz0i81zvh46skb6n3086";
- };
- doCheck = false;
- buildInputs = with pythonPackages; [
- lz4
- txrequests
- pyjade
- boto3
- moto
- txgithub
- mock
- setuptoolsTrial
- isort
- pylint
- astroid
- pyflakes
- pyjwt
- ];
-
- propagatedBuildInputs = with pythonPackages; [
-
- # core
- twisted
- jinja2
- zope_interface
- future
- sqlalchemy
- sqlalchemy_migrate
- future
- dateutil
- txaio
- autobahn
-
- # tls
- pyopenssl
- service-identity
- idna
- pkgs.treq
-
- # docs
- sphinx
- sphinxcontrib-blockdiag
- sphinxcontrib-spelling
- pyenchant
- docutils
- ramlfications
- sphinx-jinja
-
- ] ++ plugins;
-
- preInstall = ''
- # writes out a file that can't be read properly
- sed -i.bak -e '69,84d' buildbot/test/unit/test_www_config.py
-
- # re-hardcode path to tail
- sed -i.bak 's|/usr/bin/tail|${coreutils}/bin/tail|' buildbot/scripts/logwatcher.py
- '';
-
- postFixup = ''
- mv -v $out/bin/buildbot $out/bin/.wrapped-buildbot
- echo "#!/bin/sh" > $out/bin/buildbot
- echo "export PYTHONPATH=$PYTHONPATH" >> $out/bin/buildbot
- echo "exec $out/bin/.wrapped-buildbot \"\$@\"" >> $out/bin/buildbot
- chmod -c 555 $out/bin/buildbot
- '';
-
- meta = with stdenv.lib; {
- homepage = http://buildbot.net/;
- description = "Continuous integration system that automates the build/test cycle";
- maintainers = with maintainers; [ nand0p ryansydnor ];
- platforms = platforms.all;
- license = licenses.gpl2;
- };
-})
diff --git a/krebs/5pkgs/simple/buildbot/worker.nix b/krebs/5pkgs/simple/buildbot/worker.nix
deleted file mode 100644
index 34e526858..000000000
--- a/krebs/5pkgs/simple/buildbot/worker.nix
+++ /dev/null
@@ -1,24 +0,0 @@
-{ pkgs, stdenv, fetchurl, pythonPackages }:
-pythonPackages.buildPythonApplication (rec {
- name = "${pname}-${version}";
- pname = "buildbot-worker";
- version = "0.9.4";
-
- doCheck = false;
- src = fetchurl {
- url = "mirror://pypi/b/${pname}/${name}.tar.gz";
- sha256 = "0rdrr8x7sn2nxl51p6h9ad42s3c28lb6sys84zrg0d7fm4zhv7hj";
- };
-
- buildInputs = with pythonPackages; [ setuptoolsTrial mock ];
- propagatedBuildInputs = with pythonPackages; [ twisted future pkgs.treq ];
-
- meta = with stdenv.lib; {
- homepage = http://buildbot.net/;
- description = "Buildbot Worker Daemon";
- maintainers = with maintainers; [ nand0p ryansydnor ];
- platforms = platforms.all;
- license = licenses.gpl2;
- };
-})
-
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-07 17:22:46 +0000