diff options
37 files changed, 880 insertions, 475 deletions
@@ -35,7 +35,7 @@ ifeq ($(filter),json) else filter() { cat; } endif - nix-instantiate \ + result=$$(nix-instantiate \ $${extraArgs-} \ --eval \ -A "$$get" \ @@ -45,8 +45,9 @@ endif --argstr current-host-name "$$HOSTNAME" \ --argstr current-user-name "$$LOGNAME" \ $${system+--argstr system "$$system"} \ - $${target+--argstr target "$$target"} \ - | filter + $${target+--argstr target "$$target"}) + echo "$$result" | filter + else $(error unbound variable: system[s]) endif diff --git a/krebs/3modules/Reaktor.nix b/krebs/3modules/Reaktor.nix index 0fca52203..607eb7cac 100644 --- a/krebs/3modules/Reaktor.nix +++ b/krebs/3modules/Reaktor.nix @@ -9,6 +9,7 @@ let ${cfg.overrideConfig} '' else ""} ## Extra Config + ${concatStringsSep "\n" (map (plug: plug.config) cfg.plugins)} ${cfg.extraConfig} ''; cfg = config.krebs.Reaktor; @@ -35,7 +36,6 @@ let ''; }; - overrideConfig = mkOption { default = null; type = types.nullOr types.str; @@ -44,6 +44,9 @@ let Reaktor default cfg can be retrieved via `reaktor get-config` ''; }; + plugins = mkOption { + default = [pkgs.ReaktorPlugins.nixos-version]; + }; extraConfig = mkOption { default = ""; type = types.string; @@ -51,6 +54,14 @@ let configuration appended to the default or overridden configuration ''; }; + + workdir = mkOption { + default = "/var/lib/Reaktor"; + type = types.str; + description = '' + Reaktor working directory + ''; + }; extraEnviron = mkOption { default = {}; type = types.attrsOf types.str; @@ -79,7 +90,7 @@ let name = "Reaktor"; uid = genid name; description = "Reaktor user"; - home = "/var/lib/Reaktor"; + home = cfg.workdir; createHome = true; }; @@ -101,6 +112,7 @@ let GIT_SSL_CAINFO = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; REAKTOR_NICKNAME = cfg.nickname; REAKTOR_DEBUG = (if cfg.debug then "True" else "False"); + state_dir = cfg.workdir; } // cfg.extraEnviron; serviceConfig= { ExecStartPre = pkgs.writeScript "Reaktor-init" '' diff --git a/krebs/3modules/buildbot/master.nix b/krebs/3modules/buildbot/master.nix new file mode 100644 index 000000000..5870c3145 --- /dev/null +++ b/krebs/3modules/buildbot/master.nix @@ -0,0 +1,375 @@ +{ config, pkgs, lib, ... }: + +with lib; +let + buildbot = pkgs.buildbot; + buildbot-master-config = pkgs.writeText "buildbot-master.cfg" '' + # -*- python -*- + from buildbot.plugins import * + import re + import json + c = BuildmasterConfig = {} + + c['slaves'] = [] + slaves = json.loads('${builtins.toJSON cfg.slaves}') + slavenames = [ s for s in slaves ] + for k,v in slaves.items(): + c['slaves'].append(buildslave.BuildSlave(k, v)) + + # TODO: configure protocols? + c['protocols'] = {'pb': {'port': 9989}} + + ####### Build Inputs + c['change_source'] = cs = [] + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Change_Source: Begin of ${n} + ${v} + #### Change_Source: End of ${n} + '') cfg.change_source )} + + ####### Build Scheduler + c['schedulers'] = sched = [] + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Schedulers: Begin of ${n} + ${v} + #### Schedulers: End of ${n} + '') cfg.scheduler )} + + ###### Builder + c['builders'] = bu = [] + + # Builder Pre: Begin + ${cfg.builder_pre} + # Builder Pre: End + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Builder: Begin of ${n} + ${v} + #### Builder: End of ${n} + '') cfg.builder )} + + + ####### Status + c['status'] = st = [] + + # If you want to configure this url, override with extraConfig + c['buildbotURL'] = "http://${config.networking.hostName}:${toString cfg.web.port}/" + + ${optionalString (cfg.web.enable) '' + from buildbot.status import html + from buildbot.status.web import authz, auth + authz_cfg=authz.Authz( + auth=auth.BasicAuth([ ("${cfg.web.username}","${cfg.web.password}") ]), + # TODO: configure harder + gracefulShutdown = False, + forceBuild = 'auth', + forceAllBuilds = 'auth', + pingBuilder = False, + stopBuild = 'auth', + stopAllBuilds = 'auth', + cancelPendingBuild = 'auth' + ) + # TODO: configure krebs.nginx + st.append(html.WebStatus(http_port=${toString cfg.web.port}, authz=authz_cfg)) + ''} + + ${optionalString (cfg.irc.enable) '' + from buildbot.status import words + irc = words.IRC("${cfg.irc.server}", "${cfg.irc.nick}", + channels=${builtins.toJSON cfg.irc.channels}, + notify_events={ + 'success': 1, + 'failure': 1, + 'exception': 1, + 'successToFailure': 1, + 'failureToSuccess': 1, + }${optionalString cfg.irc.allowForce ",allowForce=True"}) + c['status'].append(irc) + ''} + + ${ concatStringsSep "\n" + (mapAttrsToList (n: v: '' + #### Status: Begin of ${n} + ${v} + #### Status: End of ${n} + '') cfg.status )} + + ####### PROJECT IDENTITY + c['title'] = "${cfg.title}" + c['titleURL'] = "http://krebsco.de" + + + ####### DB URL + # TODO: configure + c['db'] = { + 'db_url' : "sqlite:///state.sqlite", + } + ${cfg.extraConfig} + ''; + + cfg = config.krebs.buildbot.master; + + api = { + enable = mkEnableOption "Buildbot Master"; + title = mkOption { + default = "Buildbot CI"; + type = types.str; + description = '' + Title of the Buildbot Installation + ''; + }; + workDir = mkOption { + default = "/var/lib/buildbot/master"; + type = types.str; + description = '' + Path to build bot master directory. + Will be created on startup. + ''; + }; + + slaves = mkOption { + default = {}; + type = types.attrsOf types.str; + description = '' + Attrset of slavenames with their passwords + slavename = slavepassword + ''; + }; + + change_source = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + stockholm = '' + cs.append(changes.GitPoller( + 'http://cgit.gum/stockholm', + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + }; + description = '' + Attrset of all the change_sources which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to <literal>cs</literal> + ''; + }; + + scheduler = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + }; + description = '' + Attrset of all the schedulers which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to <literal>sched</literal> + ''; + }; + + builder_pre = mkOption { + default = ""; + type = types.lines; + example = '' + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + ''; + description = '' + some code before the builders are being assembled. + can be used to define functions used by multiple builders + ''; + }; + + builder = mkOption { + default = {}; + type = types.attrsOf types.str; + example = { + fast-test = '' + ''; + }; + description = '' + Attrset of all the builder which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to <literal>bu</literal> + ''; + }; + + status = mkOption { + default = {}; + type = types.attrsOf types.str; + description = '' + Attrset of all the extra status which should be configured. + It will be directly included into the master configuration. + + At the end an change object should be appended to <literal>st</literal> + + Right now IRC and Web status can be configured by setting + <literal>buildbot.master.irc.enable</literal> and + <literal>buildbot.master.web.enable</literal> + ''; + }; + + # Configurable Stati + web = mkOption { + default = {}; + type = types.submodule ({ config2, ... }: { + options = { + enable = mkEnableOption "Buildbot Master Web Status"; + username = mkOption { + default = "krebs"; + type = types.str; + description = '' + username for web authentication + ''; + }; + hostname = mkOption { + default = config.networking.hostName; + type = types.str; + description = '' + web interface Hostname + ''; + }; + password = mkOption { + default = "bob"; + type = types.str; + description = '' + password for web authentication + ''; + }; + port = mkOption { + default = 8010; + type = types.int; + description = '' + port for buildbot web status + ''; + }; + }; + }); + }; + + irc = mkOption { + default = {}; + type = types.submodule ({ config, ... }: { + options = { + enable = mkEnableOption "Buildbot Master IRC Status"; + channels = mkOption { + default = [ "nix-buildbot-meetup" ]; + type = with types; listOf str; + description = '' + irc channels the bot should connect to + ''; + }; + allowForce = mkOption { + default = false; + type = types.bool; + description = '' + Determines if builds can be forced via IRC + ''; + }; + nick = mkOption { + default = "nix-buildbot"; + type = types.str; + description = '' + nickname for IRC + ''; + }; + server = mkOption { + default = "irc.freenode.net"; + type = types.str; + description = '' + Buildbot Status IRC Server to connect to + ''; + }; + }; + }); + }; + + extraConfig = mkOption { + default = ""; + type = types.lines; + description = '' + extra config appended to the generated master.cfg + ''; + }; + }; + + imp = { + + users.extraUsers.buildbotMaster = { + uid = genid "buildbotMaster"; + description = "Buildbot Master"; + home = cfg.workDir; + createHome = false; + }; + + users.extraGroups.buildbotMaster = { + gid = 672626386; + }; + + systemd.services.buildbotMaster = { + description = "Buildbot Master"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + # TODO: add extra dependencies to master like svn and cvs + path = [ pkgs.git ]; + environment = { + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + }; + serviceConfig = let + workdir="${lib.shell.escape cfg.workDir}"; + secretsdir="${lib.shell.escape (toString <secrets>)}"; + in { + PermissionsStartOnly = true; + Type = "forking"; + PIDFile = "${workdir}/twistd.pid"; + # TODO: maybe also prepare buildbot.tac? + ExecStartPre = pkgs.writeScript "buildbot-master-init" '' + #!/bin/sh + set -efux + if [ ! -e ${workdir} ];then + mkdir -p ${workdir} + ${buildbot}/bin/buildbot create-master -r -l 10 -f ${workdir} + fi + # always override the master.cfg + cp ${buildbot-master-config} ${workdir}/master.cfg + # copy secrets + cp ${secretsdir}/cac.json ${workdir} + cp ${secretsdir}/retiolum-ci.rsa_key.priv \ + ${workdir}/retiolum.rsa_key.priv + # sanity + ${buildbot}/bin/buildbot checkconfig ${workdir} + + # TODO: maybe upgrade? not sure about this + # normally we should write buildbot.tac by our own + # ${buildbot}/bin/buildbot upgrade-master ${workdir} + + chmod 700 -R ${workdir} + chown buildbotMaster:buildbotMaster -R ${workdir} + ''; + ExecStart = "${buildbot}/bin/buildbot start ${workdir}"; + ExecStop = "${buildbot}/bin/buildbot stop ${workdir}"; + ExecReload = "${buildbot}/bin/buildbot reconfig ${workdir}"; + PrivateTmp = "true"; + User = "buildbotMaster"; + Restart = "always"; + RestartSec = "10"; + }; + }; + }; +in +{ + options.krebs.buildbot.master = api; + config = mkIf cfg.enable imp; +} diff --git a/makefu/3modules/buildbot/slave.nix b/krebs/3modules/buildbot/slave.nix index 7c9ea79c0..0e7796d8a 100644 --- a/makefu/3modules/buildbot/slave.nix +++ b/krebs/3modules/buildbot/slave.nix @@ -39,7 +39,7 @@ let s.setServiceParent(application) ''; default-packages = [ pkgs.git pkgs.bash ]; - cfg = config.makefu.buildbot.slave; + cfg = config.krebs.buildbot.slave; api = { enable = mkEnableOption "Buildbot Slave"; @@ -144,6 +144,7 @@ let path = default-packages ++ cfg.packages; environment = { + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; NIX_REMOTE="daemon"; } // cfg.extraEnviron; @@ -180,6 +181,6 @@ let }; in { - options.makefu.buildbot.slave = api; + options.krebs.buildbot.slave = api; config = mkIf cfg.enable imp; } diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index deccecc48..ba1f425d9 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -10,6 +10,8 @@ let ./backup.nix ./bepasty-server.nix ./build.nix + ./buildbot/master.nix + ./buildbot/slave.nix ./current.nix ./exim-retiolum.nix ./exim-smarthost.nix diff --git a/krebs/5pkgs/Reaktor/plugins.nix b/krebs/5pkgs/Reaktor/plugins.nix new file mode 100644 index 000000000..5c7b89f5c --- /dev/null +++ b/krebs/5pkgs/Reaktor/plugins.nix @@ -0,0 +1,118 @@ +{ stdenv, lib, pkgs, makeWrapper }: + +rec { + # Begin API + buildBaseReaktorPlugin = { name + , config # python extra configuration for plugin + , phases ? [] + , ... } @ attrs: + stdenv.mkDerivation (attrs // { + name = "Reaktor-plugin-" + name; + isReaktorPlugin = true; + }); + + buildSimpleReaktorPlugin = name: { script + , path ? [] + , env ? {} + , pattern ? "" + , ... } @ attrs: + let + path_env = { "PATH" = lib.makeSearchPath "bin" (path ++ [ pkgs.coreutils ]); }; + src_dir = pkgs.substituteAll ( { + inherit name; + dir = "bin"; + isExecutable = true; + src = script; + }); + src_file = "${src_dir}/bin/${name}"; + config = '' + public_commands.insert(0,{ + 'capname' : "${name}", + 'pattern' : ${if pattern == "" then + ''indirect_pattern.format("${name}")'' else + ''"${pattern}"'' }, + 'argv' : ["${src_file}"], + 'env' : ${builtins.toJSON (path_env // env)} }) + ''; + config_file = pkgs.writeText "plugin.py" config; + in buildBaseReaktorPlugin (attrs // rec { + inherit name config; + + phases = [ "installPhase" ]; + buildInputs = [ makeWrapper ]; + installPhase = '' + mkdir -p $out/bin $out/etc/Reaktor + ln -s ${src_file} $out/bin + wrapProgram $out/bin/${name} \ + --prefix PATH : ${path_env.PATH} + ln -s ${config_file} $out/etc/Reaktor/plugin.py + ''; + + }); + # End API + + # Begin Plugins + random-emoji = buildSimpleReaktorPlugin "emoji" { + path = with pkgs; [ gnused gnugrep xmlstarlet curl ]; + script = ./scripts/random-emoji.sh; + }; + + sed-plugin = buildSimpleReaktorPlugin "sed-plugin" { + path = [ pkgs.gnused ]; + # only support s///gi the plugin needs to see every msg + # TODO: this will eat up the last regex, fix Reaktor to support fallthru + pattern = "^(?P<args>.*)$$"; + script = ./scripts/sed-plugin.py; + }; + + shack-correct = buildSimpleReaktorPlugin "shack-correct" { + path = [ pkgs.gnused ]; + pattern = "^(?P<args>.*Shack.*)$$"; + script = ./scripts/shack-correct.sh; + }; + + nixos-version = buildSimpleReaktorPlugin "nixos-version" { + script = pkgs.writeScript "nixos-version" '' + #! /bin/sh + . /etc/os-release + echo "$PRETTY_NAME" + ''; + }; + stockholm-issue = buildSimpleReaktorPlugin "stockholm-issue" { + script = ./scripts/random-issue.sh; + path = with pkgs; [ git gnused lentil ]; + env = { "origin" = "http://cgit.gum/stockholm"; }; + }; + + titlebot = + let + pypkgs = pkgs.python3Packages; + titlebot_cmds = pypkgs.buildPythonPackage { + name = "titlebot_cmds"; + propagatedBuildInputs = with pypkgs; [ setuptools ]; + src = pkgs.fetchurl { + url = "https://github.com/makefu/reaktor-titlebot/archive/2.1.0.tar.gz"; + sha256 = "0wvf09wmk8b52f9j65qrw81nwrhs9pfhijwrlkzp5l7l2q8cjkp6"; + }; + }; + in buildBaseReaktorPlugin rec { + name = "titlebot"; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out + ln -s ${titlebot_cmds}/* $out + ''; + config = '' + def titlebot_cmd(cmd): + from os import environ + return { 'capname': cmd, + 'env': { 'TITLEDB': + environ['state_dir']+'/suggestions.json' }, + 'pattern': '^\\.' + cmd + '\\s*(?:\\s+(?P<args>.*))?$$', + 'argv': [ '${titlebot_cmds}/bin/' + cmd ] } + for i in ['up','help','list','top','new']: + public_commands.insert(0,titlebot_cmd(i)) + commands.insert(0,titlebot_cmd('clear')) + ''; + }; +} diff --git a/makefu/2configs/Reaktor/random-emoji.sh b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh index 386aa68b9..386aa68b9 100644 --- a/makefu/2configs/Reaktor/random-emoji.sh +++ b/krebs/5pkgs/Reaktor/scripts/random-emoji.sh diff --git a/makefu/2configs/Reaktor/random-issue.sh b/krebs/5pkgs/Reaktor/scripts/random-issue.sh index 5c47c6156..5c47c6156 100644 --- a/makefu/2configs/Reaktor/random-issue.sh +++ b/krebs/5pkgs/Reaktor/scripts/random-issue.sh diff --git a/makefu/2configs/Reaktor/sed-plugin.py b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py index 8103c9585..8103c9585 100644 --- a/makefu/2configs/Reaktor/sed-plugin.py +++ b/krebs/5pkgs/Reaktor/scripts/sed-plugin.py diff --git a/makefu/2configs/Reaktor/shack-correct.sh b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh index 3b4d04f80..3b4d04f80 100644 --- a/makefu/2configs/Reaktor/shack-correct.sh +++ b/krebs/5pkgs/Reaktor/scripts/shack-correct.sh diff --git a/krebs/5pkgs/cacpanel/default.nix b/krebs/5pkgs/cacpanel/default.nix index 3e3e2e1fc..3df4dffed 100644 --- a/krebs/5pkgs/cacpanel/default.nix +++ b/krebs/5pkgs/cacpanel/default.nix @@ -2,11 +2,11 @@ python3Packages.buildPythonPackage rec { name = "cacpanel-${version}"; - version = "0.2.1"; + version = "0.2.3"; src = pkgs.fetchurl { url = "https://pypi.python.org/packages/source/c/cacpanel/cacpanel-${version}.tar.gz"; - sha256 = "1zaazg5r10kgva32zh4fhpw6l6h51ijkwpa322na0kh4x6f6aqj3"; + sha256 = "1fib7416qqv8yzrj75kxra7ccpz9abqh58b6gkaavws2fa6m3mm8"; }; propagatedBuildInputs = with python3Packages; [ diff --git a/krebs/5pkgs/default.nix b/krebs/5pkgs/default.nix index 7df7b7d3c..c4b1dafe4 100644 --- a/krebs/5pkgs/default.nix +++ b/krebs/5pkgs/default.nix @@ -26,6 +26,8 @@ subdirs // rec { inherit (subdirs) get jq; }; + ReaktorPlugins = pkgs.callPackage ./Reaktor/plugins.nix {}; + execve = name: { filename, argv, envp ? {}, destination ? "" }: writeC name { inherit destination; } '' #include <unistd.h> @@ -40,6 +42,10 @@ subdirs // rec { } ''; + test = { + infest-cac-centos7 = pkgs.callPackage ./test/infest-cac-centos7 {}; + }; + execveBin = name: cfg: execve name (cfg // { destination = "/bin/${name}"; }); writeC = name: { destination ? "" }: src: pkgs.runCommand name {} '' diff --git a/krebs/5pkgs/test/infest-cac-centos7/default.nix b/krebs/5pkgs/test/infest-cac-centos7/default.nix new file mode 100644 index 000000000..7f2e3f231 --- /dev/null +++ b/krebs/5pkgs/test/infest-cac-centos7/default.nix @@ -0,0 +1,39 @@ +{ stdenv, coreutils,makeWrapper, cac, cacpanel, gnumake, gnused, jq, openssh, ... }: + +stdenv.mkDerivation rec { + name = "${shortname}-${version}"; + shortname = "infest-cac-centos7"; + version = "0.2.0"; + + src = ./notes; + + phases = [ + "installPhase" + ]; + buildInputs = [ makeWrapper ]; + + path = stdenv.lib.makeSearchPath "bin" [ + coreutils + cac + cacpanel + gnumake + gnused + jq + openssh + ]; + + installPhase = + '' + mkdir -p $out/bin + cp ${src} $out/bin/${shortname} + chmod +x $out/bin/${shortname} + wrapProgram $out/bin/${shortname} \ + --prefix PATH : ${path} + ''; + meta = with stdenv.lib; { + homepage = http://krebsco.de; + description = "Krebs CI Scripts"; + license = licenses.wtfpl; + maintainers = [ maintainers.makefu ]; + }; +} diff --git a/krebs/5pkgs/test/infest-cac-centos7/notes b/krebs/5pkgs/test/infest-cac-centos7/notes new file mode 100755 index 000000000..3f4fcd859 --- /dev/null +++ b/ |