From f7d979b21fc0a705105adbbc708645f94af6629c Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 4 Feb 2016 02:48:28 +0100 Subject: s 1 wolf: provide cgit mirror --- shared/2configs/cgit-mirror.nix | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 shared/2configs/cgit-mirror.nix (limited to 'shared/2configs') diff --git a/shared/2configs/cgit-mirror.nix b/shared/2configs/cgit-mirror.nix new file mode 100644 index 000000000..5bcfc5818 --- /dev/null +++ b/shared/2configs/cgit-mirror.nix @@ -0,0 +1,41 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + rules = with git;[{ + # user = git-sync; + user = git-sync; + repo = [ stockholm-mirror ]; + perm = push ''refs/*'' [ non-fast-forward create delete merge ]; + }]; + + stockholm-mirror = { + public = true; + name = "stockholm-mirror"; + desc = "mirror for all stockholm branches"; + hooks = { + post-receive = pkgs.git-hooks.irc-announce { + nick = config.networking.hostName; + verbose = false; + channel = "#retiolum"; + server = "cd.retiolum"; + }; + }; + }; + + git-sync = { + name = "git-sync"; + mail = "spam@krebsco.de"; + # TODO put git-sync pubkey somewhere more appropriate + pubkey = ''ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCzUuzyoAhMgJmsiaTVWNSXqcrZNTpKpv0nfFBOMcNXUWEbvfAq5eNpg5cX+P8eoYl6UQgfftbYi06flKK3yJdntxoZKLwJGgJt9NZr8yZTsiIfMG8XosvGNQtGPkBtpLusgmPpu7t2RQ9QrqumBvoUDGYEauKTslLwupp1QeyWKUGEhihn4CuqQKiPrz+9vbNd75XOfVZMggk3j4F7HScatmA+p1EQXWyq5Jj78jQN5ZIRnHjMQcIZ4DOz1U96atwSKMviI1xEZIODYfgoGjjiWYeEtKaLVPtSqtLRGI7l+RNouMfwHLdTWOJSlIdFncfPXC6R19hTll3UHeHLtqLP git-sync''; + }; + +in { + krebs.git = { + enable = true; + root-title = "Shared Repos"; + root-desc = "keep on krebsing"; + inherit rules; + repos.stockholm-mirror = stockholm-mirror; + }; +} -- cgit v1.2.3 From 65977c6108d9517d58a6bd6ce8676c6a7b97615e Mon Sep 17 00:00:00 2001 From: tv Date: Thu, 4 Feb 2016 04:44:26 +0100 Subject: RIP current-date --- shared/2configs/buildbot-standalone.nix | 2 -- 1 file changed, 2 deletions(-) (limited to 'shared/2configs') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix index c614bd3c1..9982dd915 100644 --- a/shared/2configs/buildbot-standalone.nix +++ b/shared/2configs/buildbot-standalone.nix @@ -86,7 +86,6 @@ -I stockholm=. \ --show-trace \ -I secrets=. '' \ - --argstr current-date lol \ --argstr current-user-name shared \ --argstr current-host-name lol \ --strict --json"]) @@ -98,7 +97,6 @@ -I stockholm=. \ -I secrets=. '' \ --show-trace \ - --argstr current-date lol \ --argstr current-user-name shared \ --argstr current-host-name lol \ --strict --json"]) -- cgit v1.2.3 From 5be8920fb0262ff703f23ef484c59f4b55a9b015 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 4 Feb 2016 14:36:48 +0100 Subject: s 2 base: new paths, cosmetics --- shared/2configs/base.nix | 18 +++++++----------- shared/2configs/cgit-mirror.nix | 7 +++---- 2 files changed, 10 insertions(+), 15 deletions(-) (limited to 'shared/2configs') diff --git a/shared/2configs/base.nix b/shared/2configs/base.nix index 5e6072661..dd698ba97 100644 --- a/shared/2configs/base.nix +++ b/shared/2configs/base.nix @@ -16,20 +16,16 @@ with lib; # TODO rename shared user to "krebs" krebs.build.user = mkDefault config.krebs.users.shared; krebs.build.source = { - git.nixpkgs = { + upstream-nixpkgs = mkDefault { url = https://github.com/NixOS/nixpkgs; rev = "d0e3cca"; - target-path = "/var/src/nixpkgs"; - }; - dir.secrets = { - host = config.krebs.current.host; - path = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}"; - }; - dir.stockholm = { - host = config.krebs.current.host; - path = mkDefault "${getEnv "HOME"}/stockholm"; - target-path = "/var/src/stockholm"; }; + secrets = mkDefault "${getEnv "HOME"}/secrets/krebs/${config.krebs.build.host.name}"; + stockholm = mkDefault "${getEnv "HOME"}/stockholm"; + + nixos-config = "symlink:stockholm/${config.krebs.build.user.name}/1systems/${config.krebs.build.host.name}.nix"; + nixpkgs = symlink:stockholm/nixpkgs; + stockholm-user = "symlink:stockholm/${config.krebs.build.user.name}"; }; networking.hostName = config.krebs.build.host.name; diff --git a/shared/2configs/cgit-mirror.nix b/shared/2configs/cgit-mirror.nix index 5bcfc5818..4ff1902f9 100644 --- a/shared/2configs/cgit-mirror.nix +++ b/shared/2configs/cgit-mirror.nix @@ -2,12 +2,11 @@ with lib; let - rules = with git;[{ - # user = git-sync; - user = git-sync; + rules = with git; singleton { + user = [ git-sync ]; repo = [ stockholm-mirror ]; perm = push ''refs/*'' [ non-fast-forward create delete merge ]; - }]; + }; stockholm-mirror = { public = true; -- cgit v1.2.3 From 4c23e33dea4d9901b64bf287983c43862f4990f2 Mon Sep 17 00:00:00 2001 From: makefu Date: Thu, 4 Feb 2016 16:38:21 +0100 Subject: ma 1: refactor buildbot config, add documentation --- shared/2configs/buildbot-standalone.nix | 150 -------------------------------- shared/2configs/shared-buildbot.nix | 148 +++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 150 deletions(-) delete mode 100644 shared/2configs/buildbot-standalone.nix create mode 100644 shared/2configs/shared-buildbot.nix (limited to 'shared/2configs') diff --git a/shared/2configs/buildbot-standalone.nix b/shared/2configs/buildbot-standalone.nix deleted file mode 100644 index 9982dd915..000000000 --- a/shared/2configs/buildbot-standalone.nix +++ /dev/null @@ -1,150 +0,0 @@ -{ lib, config, pkgs, ... }: - -{ - networking.firewall.allowedTCPPorts = [ 8010 9989 ]; - krebs.buildbot.master = { - secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; - slaves = { - testslave = "krebspass"; - }; - change_source.stockholm = '' - stockholm_repo = 'http://cgit.gum/stockholm' - cs.append(changes.GitPoller( - stockholm_repo, - workdir='stockholm-poller', branch='master', - project='stockholm', - pollinterval=120)) - ''; - scheduler = { - force-scheduler = '' - sched.append(schedulers.ForceScheduler( - name="force", - builderNames=["full-tests"])) - ''; - fast-tests-scheduler = '' - # test the master real quick - sched.append(schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - treeStableTimer=10, #only test the latest push - name="fast-master-test", - builderNames=["fast-tests"])) - ''; - test-cac-infest-master = '' - # files everyone depends on or are part of the share branch - def shared_files(change): - r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") - for file in change.files: - if r.match(file): - return True - return False - - sched.append(schedulers.SingleBranchScheduler( - change_filter=util.ChangeFilter(branch="master"), - fileIsImportant=shared_files, - treeStableTimer=60*60, # master was stable for the last hour - name="full-master-test", - builderNames=["full-tests"])) - ''; - }; - builder_pre = '' - # prepare grab_repo step for stockholm - stockholm_repo = "http://cgit.gum.retiolum/stockholm" - grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') - - env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"} - - # prepare nix-shell - # the dependencies which are used by the test script - deps = [ "gnumake", "jq","nix","rsync", - "(import {}).pkgs.test.infest-cac-centos7" ] - # TODO: --pure , prepare ENV in nix-shell command: - # SSL_CERT_FILE,LOGNAME,NIX_REMOTE - nixshell = ["nix-shell", "-I", "stockholm=.", "-p" ] + deps + [ "--run" ] - - # prepare addShell function - def addShell(factory,**kwargs): - factory.addStep(steps.ShellCommand(**kwargs)) - ''; - builder = { - fast-tests = '' - f = util.BuildFactory() - f.addStep(grab_repo) - addShell(f,name="deploy-eval-centos7",env=env, - command=nixshell + ["make -s eval get=krebs.deploy filter=json system=test-centos7"]) - - addShell(f,name="deploy-eval-wolf",env=env, - command=nixshell + ["make -s eval get=krebs.deploy filter=json system=wolf"]) - - addShell(f,name="deploy-eval-cross-check",env=env, - command=nixshell + ["! make eval get=krebs.deploy filter=json system=test-failing"]) - - addShell(f,name="instantiate-test-all-modules",env=env, - command=nixshell + \ - ["touch retiolum.rsa_key.priv; \ - nix-instantiate --eval -A \ - users.shared.test-all-krebs-modules.system \ - -I stockholm=. \ - --show-trace \ - -I secrets=. '' \ - --argstr current-user-name shared \ - --argstr current-host-name lol \ - --strict --json"]) - - addShell(f,name="instantiate-test-minimal-deploy",env=env, - command=nixshell + \ - ["nix-instantiate --eval -A \ - users.shared.test-minimal-deploy.system \ - -I stockholm=. \ - -I secrets=. '' \ - --show-trace \ - --argstr current-user-name shared \ - --argstr current-host-name lol \ - --strict --json"]) - - bu.append(util.BuilderConfig(name="fast-tests", - slavenames=slavenames, - factory=f)) - ''; - slow-tests = '' - s = util.BuildFactory() - s.addStep(grab_repo) - - # slave needs 2 files: - # * cac.json - # * retiolum - s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json")) - s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv")) - - addShell(s, name="infest-cac-centos7",env=env, - sigtermTime=60, # SIGTERM 1 minute before SIGKILL - timeout=10800, # 3h - command=nixshell + ["infest-cac-centos7"]) - - bu.append(util.BuilderConfig(name="full-tests", - slavenames=slavenames, - factory=s)) - ''; - }; - enable = true; - web = { - enable = true; - }; - irc = { - enable = true; - nick = "shared-buildbot"; - server = "cd.retiolum"; - channels = [ "retiolum" ]; - allowForce = true; - }; - }; - - krebs.buildbot.slave = { - enable = true; - masterhost = "localhost"; - username = "testslave"; - password = "krebspass"; - packages = with pkgs;[ git nix ]; - # all nix commands will need a working nixpkgs installation - extraEnviron = { NIX_PATH="nixpkgs=${toString }"; }; - }; -} diff --git a/shared/2configs/shared-buildbot.nix b/shared/2configs/shared-buildbot.nix new file mode 100644 index 000000000..50b279036 --- /dev/null +++ b/shared/2configs/shared-buildbot.nix @@ -0,0 +1,148 @@ +{ lib, config, pkgs, ... }: +# The buildbot config is seilf-contained and provides a way to test "shared" +# configuration (infrastructure to be used by every krebsminister). + +# You can add your own test, test steps as required. Deploy the config on a +# shared host like wolf and everything should be fine. +{ + networking.firewall.allowedTCPPorts = [ 8010 9989 ]; + krebs.buildbot.master = { + secrets = [ "retiolum-ci.rsa_key.priv" "cac.json" ]; + slaves = { + testslave = "krebspass"; + }; + change_source.stockholm = '' + stockholm_repo = 'http://cgit.gum/stockholm' + cs.append(changes.GitPoller( + stockholm_repo, + workdir='stockholm-poller', branch='master', + project='stockholm', + pollinterval=120)) + ''; + scheduler = { + force-scheduler = '' + sched.append(schedulers.ForceScheduler( + name="force", + builderNames=["full-tests"])) + ''; + fast-tests-scheduler = '' + # test the master real quick + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + treeStableTimer=10, #only test the latest push + name="fast-master-test", + builderNames=["fast-tests"])) + ''; + test-cac-infest-master = '' + # files everyone depends on or are part of the share branch + def shared_files(change): + r =re.compile("^((krebs|shared)/.*|Makefile|default.nix)") + for file in change.files: + if r.match(file): + return True + return False + + sched.append(schedulers.SingleBranchScheduler( + change_filter=util.ChangeFilter(branch="master"), + fileIsImportant=shared_files, + treeStableTimer=60*60, # master was stable for the last hour + name="full-master-test", + builderNames=["full-tests"])) + ''; + }; + builder_pre = '' + # prepare grab_repo step for stockholm + stockholm_repo = "http://cgit.gum.retiolum/stockholm" + grab_repo = steps.Git(repourl=stockholm_repo, mode='incremental') + + env = {"LOGNAME": "shared", "NIX_REMOTE": "daemon"} + + # prepare nix-shell + # the dependencies which are used by the test script + deps = [ "gnumake", "jq","nix","rsync", + "(import {}).pkgs.test.infest-cac-centos7" ] + # TODO: --pure , prepare ENV in nix-shell command: + # SSL_CERT_FILE,LOGNAME,NIX_REMOTE + nixshell = ["nix-shell", + "-I", "stockholm=.", + "-I", "nixpkgs=/var/src/upstream-nixpkgs", + "-p" ] + deps + [ "--run" ] + + # prepare addShell function + def addShell(factory,**kwargs): + factory.addStep(steps.ShellCommand(**kwargs)) + ''; + builder = { + fast-tests = '' + f = util.BuildFactory() + f.addStep(grab_repo) + for i in [ "test-centos7", "wolf", "test-failing" ]: + addShell(f,name="populate-{}".format(i),env=env, + command=nixshell + ["set -o pipefail;{}( nix-instantiate --arg configuration shared/1systems/{}.nix --eval --readonly-mode --show-trace -A config.krebs.build.populate --strict | jq -r .)".format("!" if "failing" in i else "",i)]) + + addShell(f,name="instantiate-test-all-modules",env=env, + command=nixshell + \ + ["touch retiolum.rsa_key.priv; \ + nix-instantiate --eval -A \ + users.shared.test-all-krebs-modules.system \ + -I stockholm=. \ + --show-trace \ + -I secrets=. '' \ + --strict --json"]) + + addShell(f,name="instantiate-test-minimal-deploy",env=env, + command=nixshell + \ + ["nix-instantiate --eval -A \ + users.shared.test-minimal-deploy.system \ + -I stockholm=. \ + -I secrets=. '' \ + --show-trace \ + --strict --json"]) + + bu.append(util.BuilderConfig(name="fast-tests", + slavenames=slavenames, + factory=f)) + ''; + slow-tests = '' + s = util.BuildFactory() + s.addStep(grab_repo) + + # slave needs 2 files: + # * cac.json + # * retiolum + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/cac.json", slavedest="cac.json")) + s.addStep(steps.FileDownload(mastersrc="${config.krebs.buildbot.master.workDir}/retiolum-ci.rsa_key.priv", slavedest="retiolum.rsa_key.priv")) + + addShell(s, name="infest-cac-centos7",env=env, + sigtermTime=60, # SIGTERM 1 minute before SIGKILL + timeout=10800, # 3h + command=nixshell + ["infest-cac-centos7"]) + + bu.append(util.BuilderConfig(name="full-tests", + slavenames=slavenames, + factory=s)) + ''; + }; + enable = true; + web = { + enable = true; + }; + irc = { + enable = true; + nick = "shared-buildbot"; + server = "cd.retiolum"; + channels = [ "retiolum" ]; + allowForce = true; + }; + }; + + krebs.buildbot.slave = { + enable = true; + masterhost = "localhost"; + username = "testslave"; + password = "krebspass"; + packages = with pkgs;[ git nix ]; + # all nix commands will need a working nixpkgs installation + extraEnviron = { NIX_PATH="/var/src"; }; + }; +} -- cgit v1.2.3