Merge branch 'master' of gum:stockholm

2 files changed, 571 insertions, 0 deletions

diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix
new file mode 100644
index 000000000..74385a433
--- /dev/null
+++ b/krebs/3modules/buildbot/master.nix
@@ -0,0 +1,385 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot = pkgs.buildbot;
+  buildbot-master-config = pkgs.writeText "buildbot-master.cfg" ''
+    # -*- python -*-
+    from buildbot.plugins import *
+    import re
+    import json
+    c = BuildmasterConfig = {}
+
+    c['slaves'] = []
+    slaves = json.loads('${builtins.toJSON cfg.slaves}')
+    slavenames = [ s for s in slaves ]
+    for k,v in slaves.items():
+      c['slaves'].append(buildslave.BuildSlave(k, v))
+
+    # TODO: configure protocols?
+    c['protocols'] = {'pb': {'port': 9989}}
+
+    ####### Build Inputs
+    c['change_source'] = cs = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Change_Source: Begin of ${n}
+        ${v}
+        #### Change_Source: End of ${n}
+      '') cfg.change_source )}
+
+    ####### Build Scheduler
+    c['schedulers'] = sched = []
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Schedulers: Begin of ${n}
+        ${v}
+        #### Schedulers: End of ${n}
+      '') cfg.scheduler )}
+
+    ###### Builder
+    c['builders'] = bu = []
+    
+    # Builder Pre: Begin
+    ${cfg.builder_pre}
+    # Builder Pre: End
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Builder: Begin of ${n}
+        ${v}
+        #### Builder: End of ${n}
+      '') cfg.builder )}
+
+
+    ####### Status
+    c['status'] = st = []
+
+    # If you want to configure this url, override with extraConfig
+    c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/"
+
+    ${optionalString (cfg.web.enable) ''
+      from buildbot.status import html
+      from buildbot.status.web import authz, auth
+      authz_cfg=authz.Authz(
+          auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]),
+          # TODO: configure harder
+          gracefulShutdown = False,
+          forceBuild = 'auth',
+          forceAllBuilds = 'auth',
+          pingBuilder = False,
+          stopBuild = 'auth',
+          stopAllBuilds = 'auth',
+          cancelPendingBuild = 'auth'
+      )
+      # TODO: configure krebs.nginx
+      st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg))
+      ''}
+
+    ${optionalString (cfg.irc.enable) ''
+      from buildbot.status import words
+      irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}",
+                      channels=${builtins.toJSON cfg.irc.channels},
+                      notify_events={
+                        'success': 1,
+                        'failure': 1,
+                        'exception': 1,
+                        'successToFailure': 1,
+                        'failureToSuccess': 1,
+                      }${optionalString cfg.irc.allowForce ",allowForce=True"})
+      c['status'].append(irc)
+      ''}
+
+    ${ concatStringsSep "\n"
+    (mapAttrsToList (n: v: ''
+        #### Status: Begin of ${n}
+        ${v}
+        #### Status: End of ${n}
+      '') cfg.status )}
+
+    ####### PROJECT IDENTITY
+    c['title'] = "${cfg.title}"
+    c['titleURL'] = "http://krebsco.de"
+
+
+    ####### DB URL
+    # TODO: configure
+    c['db'] = {
+        'db_url' : "sqlite:///state.sqlite",
+    }
+    ${cfg.extraConfig}
+    '';
+
+  cfg = config.krebs.buildbot.master;
+
+  api = {
+    enable = mkEnableOption "Buildbot Master";
+    title = mkOption {
+      default = "Buildbot CI";
+      type = types.str;
+      description = ''
+        Title of the Buildbot Installation
+      '';
+    };
+    workDir = mkOption {
+      default = "/var/lib/buildbot/master";
+      type = types.str;
+      description = ''
+        Path to build bot master directory.
+        Will be created on startup.
+      '';
+    };
+
+    secrets = mkOption {
+      default = [];
+      type = types.listOf types.str;
+      example = [ "cac.json" ];
+      description = ''
+        List of all the secrets in <secrets> which should be copied into the
+        buildbot master directory.
+      '';
+    };
+
+    slaves = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of slavenames with their passwords
+        slavename = slavepassword
+      '';
+    };
+
+    change_source = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        stockholm = ''
+          cs.append(changes.GitPoller(
+                  'http://cgit.gum/stockholm',
+                  workdir='stockholm-poller', branch='master',
+                  project='stockholm',
+                  pollinterval=120))
+        '';
+      };
+      description = ''
+        Attrset of all the change_sources which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>cs</literal>
+      '';
+    };
+
+    scheduler = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        force-scheduler = ''
+          sched.append(schedulers.ForceScheduler(
+                                      name="force",
+                                      builderNames=["full-tests"]))
+        '';
+      };
+      description = ''
+        Attrset of all the schedulers which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>sched</literal>
+      '';
+    };
+
+    builder_pre = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental')
+      '';
+      description = ''
+        some code before the builders are being assembled.
+        can be used to define functions used by multiple builders
+      '';
+    };
+
+    builder = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      example = {
+        fast-test = ''
+        '';
+      };
+      description = ''
+        Attrset of all the builder which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>bu</literal>
+      '';
+    };
+
+    status = mkOption {
+      default = {};
+      type = types.attrsOf types.str;
+      description = ''
+        Attrset of all the extra status which should be configured.
+        It will be directly included into the master configuration.
+
+        At the end an change object should be appended to <literal>st</literal>
+
+        Right now IRC and Web status can be configured by setting
+        <literal>buildbot.master.irc.enable</literal> and
+        <literal>buildbot.master.web.enable</literal>
+      '';
+    };
+
+    # Configurable Stati
+    web = mkOption {
+      default = {};
+      type = types.submodule ({ config2, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master Web Status";
+          username = mkOption {
+            default = "krebs";
+            type = types.str;
+            description = ''
+              username for web authentication
+            '';
+          };
+          hostname = mkOption {
+            default = config.networking.hostName;
+            type = types.str;
+            description = ''
+              web interface Hostname
+            '';
+          };
+          password = mkOption {
+            default = "bob";
+            type = types.str;
+            description = ''
+              password for web authentication
+            '';
+          };
+          port = mkOption {
+            default = 8010;
+            type = types.int;
+            description = ''
+              port for buildbot web status
+            '';
+          };
+        };
+      });
+    };
+
+    irc = mkOption {
+      default = {};
+      type = types.submodule ({ config, ... }: {
+        options = {
+          enable = mkEnableOption "Buildbot Master IRC Status";
+          channels = mkOption {
+            default = [ "nix-buildbot-meetup" ];
+            type = with types; listOf str;
+            description = ''
+              irc channels the bot should connect to
+            '';
+          };
+          allowForce = mkOption {
+            default = false;
+            type = types.bool;
+            description = ''
+              Determines if builds can be forced via IRC
+            '';
+          };
+          nick = mkOption {
+            default = "nix-buildbot";
+            type = types.str;
+            description = ''
+              nickname for IRC
+            '';
+          };
+          server = mkOption {
+            default = "irc.freenode.net";
+            type = types.str;
+            description = ''
+              Buildbot Status IRC Server to connect to
+            '';
+          };
+        };
+      });
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      description = ''
+        extra config appended to the generated master.cfg
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotMaster = {
+      uid = genid "buildbotMaster";
+      description = "Buildbot Master";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotMaster = {
+      gid = 672626386;
+    };
+
+    systemd.services.buildbotMaster = {
+      description = "Buildbot Master";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      # TODO: add extra dependencies to master like svn and cvs
+      path = [ pkgs.git ];
+      environment = {
+        SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+      };
+      serviceConfig = let
+        workdir="${lib.shell.escape cfg.workDir}";
+        secretsdir="${lib.shell.escape (toString <secrets>)}";
+      in {
+        PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          if [ ! -e ${workdir} ];then
+            mkdir -p ${workdir}
+            ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir}
+          fi
+          # always override the master.cfg
+          cp ${buildbot-master-config} ${workdir}/master.cfg
+
+          # copy secrets
+          ${ concatMapStringsSep "\n"
+            (f: "cp ${secretsdir}/${f} ${workdir}/${f}" ) cfg.secrets }
+          # sanity
+          ${buildbot}/bin/buildbot checkconfig ${workdir}
+
+          # TODO: maybe upgrade? not sure about this
+          #       normally we should write buildbot.tac by our own
+          # ${buildbot}/bin/buildbot upgrade-master ${workdir}
+
+          chmod 700 -R ${workdir}
+          chown buildbotMaster:buildbotMaster -R ${workdir}
+        '';
+        ExecStart = "${buildbot}/bin/buildbot start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildbot stop ${workdir}";
+        ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}";
+        PrivateTmp = "true";
+        User = "buildbotMaster";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.krebs.buildbot.master = api;
+  config = mkIf cfg.enable imp;
+}
diff --git a/krebs/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix
new file mode 100644
index 000000000..0e7796d8a
--- /dev/null
+++ b/krebs/3modules/buildbot/slave.nix
@@ -0,0 +1,186 @@
+{ config, pkgs, lib, ... }:
+
+with lib;
+let
+  buildbot-slave-init = pkgs.writeText "buildbot-slave.tac" ''
+    import os
+
+    from buildslave.bot import BuildSlave
+    from twisted.application import service
+
+    basedir = '${cfg.workDir}'
+    rotateLength = 10000000
+    maxRotatedFiles = 10
+
+    application = service.Application('buildslave')
+
+    from twisted.python.logfile import LogFile
+    from twisted.python.log import ILogObserver, FileLogObserver
+    logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength,
+                                  maxRotatedFiles=maxRotatedFiles)
+    application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+
+    buildmaster_host = '${cfg.masterhost}'
+    # TODO: masterport?
+    port = 9989
+    slavename = '${cfg.username}'
+    passwd = '${cfg.password}'
+    keepalive = 600
+    usepty = 0
+    umask = None
+    maxdelay = 300
+    allow_shutdown = None
+
+    ${cfg.extraConfig}
+
+    s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
+                  keepalive, usepty, umask=umask, maxdelay=maxdelay,
+                  allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+    '';
+  default-packages = [ pkgs.git pkgs.bash ];
+  cfg = config.krebs.buildbot.slave;
+
+  api = {
+    enable = mkEnableOption "Buildbot Slave";
+
+    workDir = mkOption {
+      default = "/var/lib/buildbot/slave";
+      type = types.str;
+      description = ''
+        Path to build bot slave directory.
+        Will be created on startup.
+      '';
+    };
+
+    masterhost = mkOption {
+      default = "localhost";
+      type = types.str;
+      description = ''
+        Hostname/IP of the buildbot master
+      '';
+    };
+
+    username = mkOption {
+      type = types.str;
+      description = ''
+        slavename used to authenticate with master
+      '';
+    };
+
+    password = mkOption {
+      type = types.str;
+      description = ''
+        slave password used to authenticate with master
+      '';
+    };
+
+    contact = mkOption {
+      default = "nix slave <buildslave@${config.networking.hostName}>";
+      type = types.str;
+      description = ''
+        contact to be announced by buildslave
+      '';
+    };
+
+    description = mkOption {
+      default = "Nix Generated BuildSlave";
+      type = types.str;
+      description = ''
+        description for hostto be announced by buildslave
+      '';
+    };
+
+    packages = mkOption {
+      default = [ pkgs.git ];
+      type = with types; listOf package;
+      description = ''
+        packages which should be in path for buildslave
+      '';
+    };
+
+    extraEnviron = mkOption {
+      default = {};
+      example = {
+        NIX_PATH = "nixpkgs=/path/to/my/nixpkgs";
+      };
+      type = types.attrsOf types.str;
+      description = ''
+        extra environment variables to be provided to the buildslave service
+        if you need nixpkgs, e.g. for running nix-shell you can set NIX_PATH here.
+      '';
+    };
+
+    extraConfig = mkOption {
+      default = "";
+      type = types.lines;
+      example = ''
+        port = 443
+        keepalive = 600
+      '';
+      description = ''
+        extra config evaluated before calling BuildSlave init in .tac file
+      '';
+    };
+  };
+
+  imp = {
+
+    users.extraUsers.buildbotSlave = {
+      uid = genid "buildbotSlave";
+      description = "Buildbot Slave";
+      home = cfg.workDir;
+      createHome = false;
+    };
+
+    users.extraGroups.buildbotSlave = {
+      gid = 1408105834;
+    };
+
+    systemd.services."buildbotSlave-${cfg.username}-${cfg.masterhost}" = {
+      description = "Buildbot Slave for ${cfg.username}@${cfg.masterhost}";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = default-packages ++ cfg.packages;
+
+      environment = {
+          SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+          NIX_REMOTE="daemon";
+      } // cfg.extraEnviron;
+
+      serviceConfig = let
+        workdir = "${lib.shell.escape cfg.workDir}";
+        contact = "${lib.shell.escape cfg.contact}";
+        description = "${lib.shell.escape cfg.description}";
+        buildbot = pkgs.buildbot-slave;
+        # TODO:make this
+      in {
+        PermissionsStartOnly = true;
+        Type = "forking";
+        PIDFile = "${workdir}/twistd.pid";
+        # TODO: maybe also prepare buildbot.tac?
+        ExecStartPre = pkgs.writeScript "buildbot-master-init" ''
+          #!/bin/sh
+          set -efux
+          mkdir -p ${workdir}/info
+          cp ${buildbot-slave-init} ${workdir}/buildbot.tac
+          echo ${contact} > ${workdir}/info/admin
+          echo ${description} > ${workdir}/info/host
+
+          chown buildbotSlave:buildbotSlave -R ${workdir}
+          chmod 700 -R ${workdir}
+        '';
+        ExecStart = "${buildbot}/bin/buildslave start ${workdir}";
+        ExecStop = "${buildbot}/bin/buildslave stop ${workdir}";
+        PrivateTmp = "true";
+        User = "buildbotSlave";
+        Restart = "always";
+        RestartSec = "10";
+      };
+    };
+  };
+in
+{
+  options.krebs.buildbot.slave = api;
+  config = mkIf cfg.enable imp;
+}