From 3371d54618aa017be77e2494c1cf82331152f3b7 Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Tue, 15 Dec 2015 18:43:40 +0100
Subject: m 3 buildbot: master init

---
 makefu/3modules/buildbot/master.nix | 179 ++++++++++++++++++++++++++++++++++++
 1 file changed, 179 insertions(+)
 create mode 100644 makefu/3modules/buildbot/master.nix

(limited to 'makefu/3modules')

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
new file mode 100644
index 000000000..310b8460d
--- /dev/null
+++ b/makefu/3modules/buildbot/master.nix
@@ -0,0 +1,179 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot = pkgs.buildbot;
+  buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
+    # -*- python -*-
+    from buildbot.plugins import *
+
+    c = BuildmasterConfig = {}
+
+    c['slaves'] = []
+    # TODO: template potential buildslaves
+    # TODO: set password?
+    for i in [ 'testslave' ]:
+      c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
+
+    c['protocols'] = {'pb': {'port': 9989}}
+
+    ####### Build Inputs
+    stockholm_repo = 'http://cgit.gum/stockholm'
+    c['change_source'] = []
+    c['change_source'].append(changes.GitPoller(
+            stockholm_repo,
+            workdir='stockholm-poller', branch='master',
+            project='stockholm',
+            pollinterval=300))
+
+    ####### Build Scheduler
+    # TODO: configure scheduler
+    important_files = util.ChangeFilter(
+                  project_re="^((krebs|share)/.*|Makefile|default.nix)",
+                  branch='master')
+    c['schedulers'] = []
+    c['schedulers'].append(schedulers.SingleBranchScheduler(
+                                name="all-important-files",
+                                change_filter=important_files,
+                                # 3 minutes stable tree
+                                treeStableTimer=3*60,
+                                builderNames=["runtests"]))
+    c['schedulers'].append(schedulers.ForceScheduler(
+                                name="force",
+                                builderNames=["runtests"]))
+    ###### The actual build
+    factory = util.BuildFactory()
+    factory.addStep(steps.Git(repourl=stockholm_repo, mode='incremental'))
+
+    deps = [ "gnumake", "jq" ]
+    factory.addStep(steps.ShellCommand(command=["nix-shell", "-p" ] + deps ))
+    factory.addStep(steps.ShellCommand(env={"LOGNAME": "shared"},
+                                       command=["make", "get=krebs.deploy",
+                                                        "system=test-centos7"]))
+
+    # TODO: different Builders?
+    c['builders'] = []
+    c['builders'].append(
+        util.BuilderConfig(name="runtests",
+          # TODO: only some slaves being used in builder?
+          slavenames=c['slaves'],
+          factory=factory))
+
+    ####### Status of Builds
+    c['status'] = []
+
+    from buildbot.status import html
+    from buildbot.status.web import authz, auth
+    # TODO: configure if http is wanted
+    authz_cfg=authz.Authz(
+        # TODO: configure user/pw
+        auth=auth.BasicAuth([("krebs","bob")]),
+        gracefulShutdown = False,
+        forceBuild = 'auth',
+        forceAllBuilds = 'auth',
+        pingBuilder = False,
+        stopBuild = False,
+        stopAllBuilds = False,
+        cancelPendingBuild = False,
+    )
+    # TODO: configure nginx
+    c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
+
+    from buildbot.status import words
+    # TODO: configure IRC Bot
+    irc = words.IRC("irc.freenode.net", "krebsbuild",
+                    channels=["krebs"],
+                    notify_events={
+                      'sucess': 1,
+                      'failure': 1,
+                      'exception': 1,
+                      'successToFailure': 1,
+                      'failureToSuccess': 1,
+                    },allowForce=True)
+    c['status'].append(irc)
+
+    ####### PROJECT IDENTITY
+    c['title'] = "Stockholm"
+    c['titleURL'] = "http://krebsco.de"
+
+    c['buildbotURL'] = "http://buildbot.krebsco.de/"
+
+    ####### DB URL
+    c['db'] = {
+        'db_url' : "sqlite:///state.sqlite",
+    }
+    ${cfg.extraConfig}
+    '';
+
+  cfg = config.makefu.buildbot.master;
+
+  api = {
+    enable = mkEnableOption "Buildbot Master";
+
+    workDir = mkOption {
+      default = "/var/lib/buildbot/master";
+      type = types.str;
+      description = ''
+        Path to build bot master directory.
+        Will be created on startup.
+      '';
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      description = ''
+        extra config appended to the generated master.cfg
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotMaster = {
+      uid = 672626386; #genid buildbotMaster
+      description = "Buildbot Master";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotMaster = {
+      gid = 672626386;
+    };
+
+    systemd.services.buildbotMaster = {
+      description = "Buildbot Master";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        PermissionsStartOnly = true;
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          workdir=${lib.shell.escape cfg.workDir}
+          if [ ! -e $workdir ];then
+            mkdir -p $workdir
+            ${buildbot}/bin/buildbot create-master -r -l 10 -f $workdir
+            chown buildbotMaster:buildbotMaster  $workdir
+          fi
+          # always override the master.cfg
+          cp ${toString buildbot-master-config} "$workdir/master.cfg"
+          # sanity
+          ${buildbot}/bin/buildbot checkconfig $workdir
+          # upgrade
+          ${buildbot}/bin/buildbot upgrade-master $workdir
+        '';
+        ExecStart = "${buildbot}/bin/buildbot ${lib.shell.escape cfg.workDir}";
+        PrivateTmp = "true";
+        User = "buildbotMaster";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.makefu.buildbot.master = api;
+  config = mkIf cfg.enable imp;
+}
-- 
cgit v1.2.3


From a907f926c120f10945c47cdaba7405fe08cfd9ee Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Tue, 15 Dec 2015 22:25:46 +0100
Subject: m 3 buildbot: first working commit for buildbot master

---
 makefu/3modules/buildbot/master.nix | 37 +++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 14 deletions(-)

(limited to 'makefu/3modules')

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
index 310b8460d..d8e917a21 100644
--- a/makefu/3modules/buildbot/master.nix
+++ b/makefu/3modules/buildbot/master.nix
@@ -12,7 +12,8 @@ let
     c['slaves'] = []
     # TODO: template potential buildslaves
     # TODO: set password?
-    for i in [ 'testslave' ]:
+    slavenames= [ 'testslave' ]
+    for i in slavenames:
       c['slaves'].append(buildslave.BuildSlave(i, "krebspass"))
 
     c['protocols'] = {'pb': {'port': 9989}}
@@ -56,7 +57,7 @@ let
     c['builders'].append(
         util.BuilderConfig(name="runtests",
           # TODO: only some slaves being used in builder?
-          slavenames=c['slaves'],
+          slavenames=slavenames,
           factory=factory))
 
     ####### Status of Builds
@@ -84,7 +85,7 @@ let
     irc = words.IRC("irc.freenode.net", "krebsbuild",
                     channels=["krebs"],
                     notify_events={
-                      'sucess': 1,
+                      'success': 1,
                       'failure': 1,
                       'exception': 1,
                       'successToFailure': 1,
@@ -145,26 +146,34 @@ let
       description = "Buildbot Master";
       after = [ "network.target" ];
       wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
+      serviceConfig = let
+        workdir="${lib.shell.escape cfg.workDir}";
+      in {
+        pidfile="${workdir}/twistd.pid";
         PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
         # TODO: maybe also prepare buildbot.tac?
         ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
           #!/bin/sh
           set -efux
-          workdir=${lib.shell.escape cfg.workDir}
-          if [ ! -e $workdir ];then
-            mkdir -p $workdir
-            ${buildbot}/bin/buildbot create-master -r -l 10 -f $workdir
-            chown buildbotMaster:buildbotMaster  $workdir
+          if [ ! -e ${workdir} ];then
+            mkdir -p ${workdir}
+            ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
           fi
           # always override the master.cfg
-          cp ${toString buildbot-master-config} "$workdir/master.cfg"
+          cp ${buildbot-master-config} ${workdir}/master.cfg
           # sanity
-          ${buildbot}/bin/buildbot checkconfig $workdir
-          # upgrade
-          ${buildbot}/bin/buildbot upgrade-master $workdir
+          ${buildbot}/bin/buildbot checkconfig ${workdir}
+
+          # TODO: maybe upgrade?
+          # ${buildbot}/bin/buildbot upgrade-master ${workdir}
+
+          chown buildbotMaster:buildbotMaster -R ${workdir}
         '';
-        ExecStart = "${buildbot}/bin/buildbot ${lib.shell.escape cfg.workDir}";
+        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
         PrivateTmp = "true";
         User = "buildbotMaster";
         Restart = "always";
-- 
cgit v1.2.3


From c95085d875ac72152dcfbaceb35364203f97db7d Mon Sep 17 00:00:00 2001
From: makefu <github@syntax-fehler.de>
Date: Wed, 16 Dec 2015 10:42:20 +0100
Subject: m 3 buildbot: add to imports

---
 makefu/3modules/default.nix | 1 +
 1 file changed, 1 insertion(+)

(limited to 'makefu/3modules')

diff --git a/makefu/3modules/default.nix b/makefu/3modules/default.nix
index a8a1f69d0..4b2b36e64 100644
--- a/makefu/3modules/default.nix
+++ b/makefu/3modules/default.nix
@@ -2,6 +2,7 @@ _:
 
 {
   imports = [
+    ./buildbot/master.nix
   ];
 }
 
-- 
cgit v1.2.3


From 20a52c8dee414e89dba49f4a4a12e20d6555c55e Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Wed, 16 Dec 2015 14:29:46 +0100
Subject: m 3 buildbot/master: make irc configurable

---
 makefu/3modules/buildbot/master.nix | 66 +++++++++++++++++++++++++++++--------
 1 file changed, 52 insertions(+), 14 deletions(-)

(limited to 'makefu/3modules')

diff --git a/makefu/3modules/buildbot/master.nix b/makefu/3modules/buildbot/master.nix
index d8e917a21..5d340f899 100644
--- a/makefu/3modules/buildbot/master.nix
+++ b/makefu/3modules/buildbot/master.nix
@@ -81,17 +81,19 @@ let
     c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
 
     from buildbot.status import words
-    # TODO: configure IRC Bot
-    irc = words.IRC("irc.freenode.net", "krebsbuild",
-                    channels=["krebs"],
-                    notify_events={
-                      'success': 1,
-                      'failure': 1,
-                      'exception': 1,
-                      'successToFailure': 1,
-                      'failureToSuccess': 1,
-                    },allowForce=True)
-    c['status'].append(irc)
+    ${optionalString (cfg.irc.enable) ''
+      irc = words.IRC("${cfg.irc.server}", "krebsbuild",
+                      # TODO: multiple channels
+                      channels=["${cfg.irc.channel}"],
+                      notify_events={
+                        'success': 1,
+                        'failure': 1,
+                        'exception': 1,
+                        'successToFailure': 1,
+                        'failureToSuccess': 1,
+                      }${optionalString cfg.irc.allowForce ",allowForce=True"})
+      c['status'].append(irc)
+      ''}
 
     ####### PROJECT IDENTITY
     c['title'] = "Stockholm"
@@ -119,7 +121,42 @@ let
         Will be created on startup.
       '';
     };
-
+    irc = mkOption {
+      default = {};
+      type = types.submodule ({ config, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master IRC Status";
+          channel = mkOption {
+            default = "nix-buildbot-meetup";
+            type = types.str;
+            description = ''
+              irc channel the bot should connect to
+            '';
+          };
+          allowForce = mkOption {
+            default = false;
+            type = types.bool;
+            description = ''
+              Determines if builds can be forced via IRC
+            '';
+          };
+          nick = mkOption {
+            default = "nix-buildbot";
+            type = types.str;
+            description = ''
+              nickname for IRC
+            '';
+          };
+          server = mkOption {
+            default = "irc.freenode.net";
+            type = types.str;
+            description = ''
+              Buildbot Status IRC Server to connect to
+            '';
+          };
+        };
+      });
+    };
     extraConfig = mkOption {
       default = "";
       type = types.lines;
@@ -149,7 +186,6 @@ let
       serviceConfig = let
         workdir="${lib.shell.escape cfg.workDir}";
       in {
-        pidfile="${workdir}/twistd.pid";
         PermissionsStartOnly = true;
         Type = "forking";
         PIDFile = "${workdir}/twistd.pid";
@@ -166,9 +202,11 @@ let
           # sanity
           ${buildbot}/bin/buildbot checkconfig ${workdir}
 
-          # TODO: maybe upgrade?
+          # TODO: maybe upgrade? not sure about this
+          #       normally we should write buildbot.tac by our own
           # ${buildbot}/bin/buildbot upgrade-master ${workdir}
 
+          chmod 700 -R ${workdir}
           chown buildbotMaster:buildbotMaster -R ${workdir}
         '';
         ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
-- 
cgit v1.2.3


From 2156aa4d37071408b8e5385a4f639ed029f70620 Mon Sep 17 00:00:00 2001
From: makefu <makefu@nixos.dev>
Date: Wed, 16 Dec 2015 14:30:01 +0100
Subject: m 3 buildbot/slave: init

---
 makefu/3modules/buildbot/slave.nix | 159 +++++++++++++++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 makefu/3modules/buildbot/slave.nix

(limited to 'makefu/3modules')

diff --git a/makefu/3modules/buildbot/slave.nix b/makefu/3modules/buildbot/slave.nix
new file mode 100644
index 000000000..188a9283c
--- /dev/null
+++ b/makefu/3modules/buildbot/slave.nix
@@ -0,0 +1,159 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
+    import os
+
+    from buildslave.bot import BuildSlave
+    from twisted.application import service
+
+    basedir = '${cfg.workDir}'
+    rotateLength = 10000000
+    maxRotatedFiles = 10
+
+    application = service.Application('buildslave')
+
+    from twisted.python.logfile import LogFile
+    from twisted.python.log import ILogObserver, FileLogObserver
+    logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+                                  maxRotatedFiles=maxRotatedFiles)
+    application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+    buildmaster_host = '${cfg.masterhost}'
+    # TODO: masterport?
+    port = 9989
+    slavename = '${cfg.username}'
+    passwd = '${cfg.password}'
+    keepalive = 600
+    usepty = 0
+    umask = None
+    maxdelay = 300
+    allow_shutdown = None
+
+    ${cfg.extraConfig}
+
+    s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
+                  keepalive, usepty, umask=umask, maxdelay=maxdelay,
+                  allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+    '';
+
+  cfg = config.makefu.buildbot.slave;
+
+  api = {
+    enable = mkEnableOption "Buildbot Slave";
+
+    workDir = mkOption {
+      default = "/var/lib/buildbot/slave";
+      type = types.str;
+      description = ''
+        Path to build bot slave directory.
+        Will be created on startup.
+      '';
+    };
+
+    masterhost = mkOption {
+      default = "localhost";
+      type = types.str;
+      description = ''
+        Hostname/IP of the buildbot master
+      '';
+    };
+
+    username = mkOption {
+      type = types.str;
+      description = ''
+        slavename used to authenticate with master
+      '';
+    };
+
+    password = mkOption {
+      type = types.str;
+      description = ''
+        slave password used to authenticate with master
+      '';
+    };
+
+    contact = mkOption {
+      default = "nix slave <buildslave@${config.networking.hostName}>";
+      type = types.str;
+      description = ''
+        contact to be announced by buildslave
+      '';
+    };
+
+    description = mkOption {
+      default = "Nix Generated BuildSlave";
+      type = types.str;
+      description = ''
+        description for hostto be announced by buildslave
+      '';
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        port = 443
+        keepalive = 600
+      '';
+      description = ''
+        extra config evaluated before calling BuildSlave init in .tac file
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotSlave = {
+      uid = 1408105834; #genid buildbotMaster
+      description = "Buildbot Slave";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotSlave = {
+      gid = 1408105834;
+    };
+
+    systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
+      description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = let
+        workdir = "${lib.shell.escape cfg.workDir}";
+        contact = "${lib.shell.escape cfg.contact}";
+        description = "${lib.shell.escape cfg.description}";
+        buildbot = pkgs.buildbot-slave;
+        # TODO:make this
+      in {
+        PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          mkdir -p ${workdir}/info
+          cp ${buildbot-slave-init} ${workdir}/buildbot.tac
+          echo ${contact} > ${workdir}/info/admin
+          echo ${description} > ${workdir}/info/host
+
+          chown buildbotSlave:buildbotSlave -R ${workdir}
+          chmod 700 -R ${workdir}
+        '';
+        ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
+        PrivateTmp = "true";
+        User = "buildbotSlave";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.makefu.buildbot.slave = api;
+  config = mkIf cfg.enable imp;
+}
-- 
cgit v1.2.3