From f13fed245b94e5e2669ab348475cc0169a01b7bf Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 23 Oct 2022 18:30:56 +0200 Subject: ci: move to extra folder, report failed scripts --- krebs/3modules/ci.nix | 199 ------------------------------ krebs/3modules/ci/default.nix | 210 ++++++++++++++++++++++++++++++++ krebs/3modules/ci/modules/irc_notify.py | 145 ++++++++++++++++++++++ krebs/3modules/default.nix | 2 +- 4 files changed, 356 insertions(+), 200 deletions(-) delete mode 100644 krebs/3modules/ci.nix create mode 100644 krebs/3modules/ci/default.nix create mode 100644 krebs/3modules/ci/modules/irc_notify.py diff --git a/krebs/3modules/ci.nix b/krebs/3modules/ci.nix deleted file mode 100644 index 5efe4178..00000000 --- a/krebs/3modules/ci.nix +++ /dev/null @@ -1,199 +0,0 @@ -{ config, lib, pkgs, ... }: - -with import ; - -let - cfg = config.krebs.ci; - - out = { - options.krebs.ci = api; - config = lib.mkIf cfg.enable imp; - }; - - api = { - enable = mkEnableOption "Enable krebs ci service"; - repos = mkOption { - type = types.attrsOf (types.submodule ({ config, ...}: { - options = { - urls = mkOption { - type = types.listOf types.str; - default = [ "git@localhost:${config._module.args.name}" ]; - }; - }; - })); - }; - }; - - hostname = config.networking.hostName; - getJobs = pkgs.writeDash "get_jobs" '' - set -efu - ${pkgs.nix}/bin/nix-build --no-out-link --quiet --show-trace -Q ./ci.nix >&2 - json="$(${pkgs.nix}/bin/nix-instantiate --quiet -Q --eval --strict --json ./ci.nix)" - echo "$json" | ${pkgs.jq}/bin/jq -r 'to_entries[] | [.key, .value] | @tsv' \ - | while read -r host builder; do - gcroot=${shell.escape profileRoot}/$host-builder - ${pkgs.nix}/bin/nix-env -p "$gcroot" --set "$builder" - done - echo "$json" - ''; - - profileRoot = "/nix/var/nix/profiles/ci"; - - bcfg = config.services.buildbot-master; - - imp = { - services.buildbot-master = { - workers = [ "worker.Worker('testworker', 'pass')" ]; - - changeSource = mapAttrsToList (name: repo: - concatMapStringsSep "," (url: '' - changes.GitPoller( - "${url}", - workdir='${name}-${elemAt(splitString "." url) 1}', branches=True, - project='${name}', - pollinterval=100 - ) - '') repo.urls - ) cfg.repos; - - schedulers = mapAttrsToList (name: repo: '' - schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter( - branch_re=".*", - project='${name}', - ), - treeStableTimer=60, - name="${name}-all-branches", - builderNames=[ - "${name}", - ] - ), - schedulers.ForceScheduler( - name="${name}", - builderNames=[ - "${name}", - ] - ) - '') cfg.repos; - - builders = []; - - extraConfig = '' - # https://docs.buildbot.net/latest/manual/configuration/buildfactories.html - from buildbot.plugins import util, steps - from buildbot.process import buildstep, logobserver - from twisted.internet import defer - import json - - class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): - def __init__(self, **kwargs): - kwargs = self.setupShellMixin(kwargs) - super().__init__(**kwargs) - self.observer = logobserver.BufferLogObserver() - self.addLogObserver('stdio', self.observer) - - def extract_stages(self, stdout): - stages = json.loads(stdout) - return stages - - @defer.inlineCallbacks - def run(self): - # run nix-instanstiate to generate the dict of stages - cmd = yield self.makeRemoteShellCommand() - yield self.runCommand(cmd) - - # if the command passes extract the list of stages - result = cmd.results() - if result == util.SUCCESS: - # create a ShellCommand for each stage and add them to the build - stages = self.extract_stages(self.observer.getStdout()) - self.build.addStepsAfterCurrentStep([ - steps.ShellCommand( - name=stage, - env=dict( - build_name = stage, - build_script = stages[stage], - ), - command="${pkgs.writeDash "build.sh" '' - set -xefu - profile=${shell.escape profileRoot}/$build_name - result=$("$build_script") - if [ -n "$result" ]; then - ${pkgs.nix}/bin/nix-env -p "$profile" --set "$result" - fi - ''}", - ) for stage in stages - ]) - - return result - - - ${concatStringsSep "\n" (mapAttrsToList (name: repo: '' - factory_${name} = util.BuildFactory() - factory_${name}.addStep(steps.Git( - repourl=util.Property('repository', '${head repo.urls}'), - method='clobber', - mode='full', - submodules=True, - )) - - factory_${name}.addStep(GenerateStagesCommand( - env={ - "NIX_REMOTE": "daemon", - "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src", - }, - name="Generate build stages", - command=[ - "${getJobs}" - ], - haltOnFailure=True, - )) - - c['builders'].append( - util.BuilderConfig( - name="${name}", - workernames=['testworker'], - factory=factory_${name} - ) - ) - '') cfg.repos)} - ''; - - enable = true; - reporters = ['' - reporters.IRC( - host = "irc.r", - nick = "buildbot|${hostname}", - notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ], - channels = [{"channel": "#xxx"}], - showBlameList = True, - authz={'force': True}, - ) - '']; - - buildbotUrl = "http://build.${hostname}.r/"; - }; - - services.buildbot-worker = { - enable = true; - workerUser = "testworker"; - workerPass = "pass"; - packages = with pkgs; [ git gnutar gzip jq nix populate ]; - }; - - system.activationScripts.buildbots-nix-profile = '' - ${pkgs.coreutils}/bin/mkdir -p ${shell.escape profileRoot} - ${pkgs.coreutils}/bin/chmod 0770 ${shell.escape profileRoot} - ${pkgs.coreutils}/bin/chgrp buildbots ${shell.escape profileRoot} - ''; - - users = { - groups.buildbots.gid = genid "buildbots"; - users = { - buildbot.extraGroups = [ "buildbots" ]; - bbworker.extraGroups = [ "buildbots" ]; - }; - }; - }; - -in out diff --git a/krebs/3modules/ci/default.nix b/krebs/3modules/ci/default.nix new file mode 100644 index 00000000..0f85b27c --- /dev/null +++ b/krebs/3modules/ci/default.nix @@ -0,0 +1,210 @@ +{ config, lib, pkgs, ... }: + +with import ; + +let + cfg = config.krebs.ci; + + out = { + options.krebs.ci = api; + config = lib.mkIf cfg.enable imp; + }; + + api = { + enable = mkEnableOption "Enable krebs ci service"; + repos = mkOption { + type = types.attrsOf (types.submodule ({ config, ...}: { + options = { + urls = mkOption { + type = types.listOf types.str; + default = [ "git@localhost:${config._module.args.name}" ]; + }; + }; + })); + }; + }; + + hostname = config.networking.hostName; + getJobs = pkgs.writeDash "get_jobs" '' + set -efu + ${pkgs.nix}/bin/nix-build --no-out-link --quiet --show-trace -Q ./ci.nix >&2 + json="$(${pkgs.nix}/bin/nix-instantiate --quiet -Q --eval --strict --json ./ci.nix)" + echo "$json" | ${pkgs.jq}/bin/jq -r 'to_entries[] | [.key, .value] | @tsv' \ + | while read -r host builder; do + gcroot=${shell.escape profileRoot}/$host-builder + ${pkgs.nix}/bin/nix-env -p "$gcroot" --set "$builder" + done + echo "$json" + ''; + + profileRoot = "/nix/var/nix/profiles/ci"; + + bcfg = config.services.buildbot-master; + + imp = { + services.buildbot-master = { + workers = [ "worker.Worker('testworker', 'pass')" ]; + + changeSource = mapAttrsToList (name: repo: + concatMapStringsSep "," (url: '' + changes.GitPoller( + "${url}", + workdir='${name}-${elemAt(splitString "." url) 1}', branches=True, + project='${name}', + pollinterval=30 + ) + '') repo.urls + ) cfg.repos; + + schedulers = mapAttrsToList (name: repo: '' + schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter( + branch_re=".*", + project='${name}', + ), + treeStableTimer=60, + name="${name}-all-branches", + builderNames=[ + "${name}", + ] + ), + schedulers.ForceScheduler( + name="${name}", + builderNames=[ + "${name}", + ] + ) + '') cfg.repos; + + builders = []; + + extraConfig = '' + # https://docs.buildbot.net/latest/manual/configuration/buildfactories.html + from buildbot.plugins import util, steps + from buildbot.process import buildstep, logobserver + from twisted.internet import defer + import json + import sys + + class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): + def __init__(self, **kwargs): + kwargs = self.setupShellMixin(kwargs) + super().__init__(**kwargs) + self.observer = logobserver.BufferLogObserver() + self.addLogObserver('stdio', self.observer) + + def extract_stages(self, stdout): + stages = json.loads(stdout) + return stages + + @defer.inlineCallbacks + def run(self): + # run nix-instanstiate to generate the dict of stages + cmd = yield self.makeRemoteShellCommand() + yield self.runCommand(cmd) + + # if the command passes extract the list of stages + result = cmd.results() + if result == util.SUCCESS: + # create a ShellCommand for each stage and add them to the build + stages = self.extract_stages(self.observer.getStdout()) + self.build.addStepsAfterCurrentStep([ + steps.ShellCommand( + name=stage, + env=dict( + build_name = stage, + build_script = stages[stage], + ), + command="${pkgs.writeDash "build.sh" '' + set -xefu + profile=${shell.escape profileRoot}/$build_name + result=$("$build_script") + if [ -n "$result" ]; then + ${pkgs.nix}/bin/nix-env -p "$profile" --set "$result" + fi + ''}", + ) for stage in stages + ]) + + return result + + + ${concatStringsSep "\n" (mapAttrsToList (name: repo: '' + factory_${name} = util.BuildFactory() + factory_${name}.addStep(steps.Git( + repourl=util.Property('repository', '${head repo.urls}'), + method='clobber', + mode='full', + submodules=True, + )) + + factory_${name}.addStep(GenerateStagesCommand( + env={ + "NIX_REMOTE": "daemon", + "NIX_PATH": "secrets=/var/src/stockholm/null:/var/src", + }, + name="Generate build stages", + command=[ + "${getJobs}" + ], + haltOnFailure=True, + )) + + c['builders'].append( + util.BuilderConfig( + name="${name}", + workernames=['testworker'], + factory=factory_${name} + ) + ) + '') cfg.repos)} + + # fancy irc notification by Mic92 https://github.com/Mic92/dotfiles/tree/master/nixos/eve/modules/buildbot + sys.path.append("${./modules}") + from irc_notify import NotifyFailedBuilds + c['services'].append( + NotifyFailedBuilds("irc://buildbot|test@irc.r:6667/#xxx") + ) + + ''; + + enable = true; + reporters = [ + '' + reporters.IRC( + host = "irc.r", + nick = "buildbot|${hostname}", + notify_events = [ 'started', 'finished', 'failure', 'success', 'exception', 'problem' ], + channels = [{"channel": "#xxx"}], + showBlameList = True, + authz={'force': True}, + ) + '' + ]; + + buildbotUrl = "http://build.${hostname}.r/"; + }; + + services.buildbot-worker = { + enable = true; + workerUser = "testworker"; + workerPass = "pass"; + packages = with pkgs; [ git gnutar gzip jq nix populate ]; + }; + + system.activationScripts.buildbots-nix-profile = '' + ${pkgs.coreutils}/bin/mkdir -p ${shell.escape profileRoot} + ${pkgs.coreutils}/bin/chmod 0770 ${shell.escape profileRoot} + ${pkgs.coreutils}/bin/chgrp buildbots ${shell.escape profileRoot} + ''; + + users = { + groups.buildbots.gid = genid "buildbots"; + users = { + buildbot.extraGroups = [ "buildbots" ]; + bbworker.extraGroups = [ "buildbots" ]; + }; + }; + }; + +in out diff --git a/krebs/3modules/ci/modules/irc_notify.py b/krebs/3modules/ci/modules/irc_notify.py new file mode 100644 index 00000000..4b7969aa --- /dev/null +++ b/krebs/3modules/ci/modules/irc_notify.py @@ -0,0 +1,145 @@ +from typing import Optional, Generator, Any +import socket +import ssl +import threading +import re +from urllib.parse import urlparse +import base64 + +from buildbot.reporters.base import ReporterBase +from buildbot.reporters.generators.build import BuildStatusGenerator +from buildbot.reporters.message import MessageFormatter +from twisted.internet import defer + +DEBUG = False + + +def _irc_send( + server: str, + nick: str, + channel: str, + sasl_password: Optional[str] = None, + server_password: Optional[str] = None, + tls: bool = True, + port: int = 6697, + messages: list[str] = [], +) -> None: + if not messages: + return + + # don't give a shit about legacy ip + sock = socket.socket(family=socket.AF_INET6) + if tls: + sock = ssl.wrap_socket( + sock, cert_reqs=ssl.CERT_NONE, ssl_version=ssl.PROTOCOL_TLSv1_2 + ) + + def _send(command: str) -> int: + if DEBUG: + print(command) + return sock.send((f"{command}\r\n").encode()) + + def _pong(ping: str): + if ping.startswith("PING"): + sock.send(ping.replace("PING", "PONG").encode("ascii")) + + recv_file = sock.makefile(mode="r") + + print(f"connect {server}:{port}") + sock.connect((server, port)) + if server_password: + _send(f"PASS {server_password}") + _send(f"USER {nick} 0 * :{nick}") + _send(f"NICK {nick}") + for line in recv_file.readline(): + if re.match(r"^:[^ ]* (MODE|221|376|422) ", line): + break + else: + _pong(line) + + if sasl_password: + _send("CAP REQ :sasl") + _send("AUTHENTICATE PLAIN") + auth = base64.encodebytes(f"{nick}\0{nick}\0{sasl_password}".encode("ascii")) + _send(f"AUTHENTICATE {auth.decode('ascii')}") + _send("CAP END") + _send(f"JOIN :{channel}") + + for m in messages: + _send(f"PRIVMSG {channel} :{m}") + + _send("INFO") + for line in recv_file: + if DEBUG: + print(line, end="") + # Assume INFO reply means we are done + if "End of /INFO" in line: + break + else: + _pong(line) + + sock.send(b"QUIT") + print("disconnect") + sock.close() + + +def irc_send( + url: str, notifications: list[str], password: Optional[str] = None +) -> None: + parsed = urlparse(f"{url}") + username = parsed.username or "prometheus" + server = parsed.hostname or "chat.freenode.net" + if parsed.fragment != "": + channel = f"#{parsed.fragment}" + else: + channel = "#krebs-announce" + port = parsed.port or 6697 + if not password: + password = parsed.password + if len(notifications) == 0: + return + # put this in a thread to not block buildbot + t = threading.Thread( + target=_irc_send, + kwargs=dict( + server=server, + nick=username, + sasl_password=password, + channel=channel, + port=port, + messages=notifications, + tls=parsed.scheme == "irc+tls", + ), + ) + t.start() + + +subject_template = """\ +{{ '☠' if result_names[results] == 'failure' else '☺' if result_names[results] == 'success' else '☝' }} \ +{{ build['properties'].get('project', ['whole buildset'])[0] if is_buildset else buildername }} \ +- \ +{{ build['state_string'] }} \ +{{ '(%s)' % (build['properties']['branch'][0] if (build['properties']['branch'] and build['properties']['branch'][0]) else build['properties'].get('got_revision', ['(unknown revision)'])[0]) }} \ +({{ build_url }}) +""" # # noqa pylint: disable=line-too-long + + +class NotifyFailedBuilds(ReporterBase): + def _generators(self) -> list[BuildStatusGenerator]: + formatter = MessageFormatter(template_type="plain", subject=subject_template) + return [BuildStatusGenerator(message_formatter=formatter)] + + def checkConfig(self, url: str): + super().checkConfig(generators=self._generators()) + + @defer.inlineCallbacks + def reconfigService(self, url: str) -> Generator[Any, object, Any]: + self.url = url + yield super().reconfigService(generators=self._generators()) + + def sendMessage(self, reports: list): + msgs = [] + for r in reports: + if r["builds"][0]["state_string"] != "build successful": + msgs.append(r["subject"]) + irc_send(self.url, notifications=msgs) diff --git a/krebs/3modules/default.nix b/krebs/3modules/default.nix index 5ba43658..01436d35 100644 --- a/krebs/3modules/default.nix +++ b/krebs/3modules/default.nix @@ -16,7 +16,7 @@ let ./brockman.nix ./build.nix ./cachecache.nix - ./ci.nix + ./ci ./current.nix ./dns.nix ./ergo.nix -- cgit v1.2.3