diff options
62 files changed, 772 insertions, 730 deletions
diff --git a/.rsync-filter b/.rsync-filter new file mode 100644 index 000000000..d7657cd00 --- /dev/null +++ b/.rsync-filter @@ -0,0 +1,2 @@ +- /.git +- /.graveyard @@ -1,53 +1,51 @@ -# -# usage: -# make infest system=foo [target=bar] -# make [deploy] system=foo [target=bar] -# make [deploy] systems='foo bar' -# make eval get=users.tv.wu.config.time.timeZone [filter=json] -# - .ONESHELL: .SHELLFLAGS := -eufc -ifdef systems -$(systems): - @ - unset target - parallel \ - --line-buffer \ - -j0 \ - --no-notice \ - --tagstring {} \ - -q make -s systems= system={} ::: $(systems) -else ifdef system -.PHONY: deploy infest -deploy infest:;@ - export get=krebs.$@ - export filter=json - make -s eval | sh +ifndef system +$(error unbound variable: system) +endif + +export target_host ?= $(system) +export target_user ?= root +export target_path ?= /var/src + +# usage: make deploy system=foo [target_host=bar] +.PHONY: deploy +deploy: populate ;@set -x + ssh "$$target_user@$$target_host" nixos-rebuild switch -I "$$target_path" +# usage: make populate system=foo [target_host=bar] +.PHONY: populate +populate:;@ + result=$$(make -s eval get=config.krebs.build.populate filter=json) + echo "$$result" | sh + +# usage: make eval system=foo get=config.krebs.build [LOGNAME=tv] [filter=json] .PHONY: eval -eval: - @ +eval:;@ ifeq ($(filter),json) extraArgs='--json --strict' - filter() { jq -r .; } + filter() { echo "$$1" | jq -r .; } else - filter() { cat; } + filter() { echo "$$1"; } endif result=$$(nix-instantiate \ $${extraArgs-} \ + --show-trace \ + --readonly-mode \ --eval \ -A "$$get" \ - -I stockholm="$$PWD" \ - '<stockholm>' \ - --argstr current-date "$$(date -Is)" \ - --argstr current-host-name "$$HOSTNAME" \ - --argstr current-user-name "$$LOGNAME" \ - $${system+--argstr system "$$system"} \ - $${target+--argstr target "$$target"}) - echo "$$result" | filter + --arg configuration "./$$LOGNAME/1systems/$$system.nix") + filter "$$result" -else -$(error unbound variable: system[s]) -endif +## usage: make install system=foo target= +#.PHONY: install +#install: ssh = ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null +#install:;@set -x +# $(ssh) "$$target_user@$$target_host" \ +# env target_path=/var/src \ +# sh -s prepare < krebs/4lib/infest/prepare.sh +# make -s populate target_path=/mnt"$$target_path" +# $(ssh) "$$target_user@$$target_host" \ +# env NIXOS_CONFIG=/var/src/nixos-config \ +# nixos-install diff --git a/default.nix b/default.nix index 1637aa464..278f1d14d 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,15 @@ -{ current-date ? abort "current-date not defined" -, current-host-name ? abort "current-host-name not defined" -, current-user-name ? builtins.getEnv "LOGNAME" +{ configuration ? import (nixpkgs-path + "/nixos/lib/from-env.nix") "NIXOS_CONFIG" <nixos-config> +, system ? builtins.currentSystem +, current-host-name ? + let v = builtins.getEnv "HOSTNAME"; in + if v != "" then v else builtins.readFile /proc/sys/kernel/hostname +, current-user-name ? + let v = builtins.getEnv "LOGNAME"; in + if v != "" then v else abort "undefined variable: LOGNAME" +, nixpkgs-path ? + if (builtins.tryEval <nixpkgs/krebs>).success + then <upstream-nixpkgs> + else <nixpkgs> , StrictHostKeyChecking ? "yes" }@args: @@ -8,26 +17,26 @@ let stockholm = { inherit krebs; inherit users; inherit lib; - inherit pkgs; + inherit config options pkgs; + system = config.system.build.toplevel; }; krebs = import ./krebs (args // { inherit lib stockholm; }); lib = let - nlib = import <nixpkgs/lib>; + nlib = import (slib.npath "lib"); klib = import (slib.kpath "4lib") { lib = nlib; }; slib = rec { - stockholm-path = ./.; - nspath = ns: p: stockholm-path + "/${ns}/${p}"; - kpath = nspath "krebs"; - upath = nspath current-user-name; + npath = p: nixpkgs-path + "/${p}"; + kpath = p: ./. + "/krebs/${p}"; + upath = p: ./. + "/${current-user-name}/${p}"; }; ulib = let p = slib.upath "4lib"; in nlib.optionalAttrs (klib.dir.has-default-nix p) (import p { lib = nlib // klib; }); in nlib // klib // slib // ulib // builtins; - inherit (eval {}) pkgs; + inherit (eval configuration) config options pkgs; base-module = { config, ... }: { imports = builtins.filter lib.dir.has-default-nix (lib.concatLists [ @@ -45,7 +54,8 @@ let stockholm = { in kpkgs // upkgs; }; - eval = config: import <nixpkgs/nixos/lib/eval-config.nix> { + eval = config: import (lib.npath "nixos/lib/eval-config.nix") { + inherit system; specialArgs = { inherit lib; }; diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix index 01bb16a2b..a1f335905 100644 --- a/krebs/3modules/backup.nix +++ b/krebs/3modules/backup.nix @@ -28,9 +28,17 @@ let type = types.krebs.file-location; }; startAt = mkOption { - type = types.str; + default = "hourly"; + type = types.str; # TODO systemd.time(7)'s calendar event }; snapshots = mkOption { + default = { + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; type = types.attrsOf (types.submodule { options = { format = mkOption { @@ -284,3 +292,15 @@ let }; in out +# TODO ionice +# TODO mail on failed push, pull +# TODO mail on missing push +# TODO don't cancel plans on activation +# also, don't hang while deploying at: +# starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer +# TODO make sure /bku is properly mounted +# TODO make sure that secure hosts cannot backup to insecure ones +# TODO optionally only backup when src and dst are near enough :) +# TODO try using btrfs for snapshots (configurable) +# TODO warn if partial snapshots are found +# TODO warn if unknown stuff is found in dst path diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 7f004cd81..00142acdd 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -28,48 +28,159 @@ let type = types.user; }; - options.krebs.build.source.dir = mkOption { - type = let - default-host = config.krebs.current.host; - in types.attrsOf (types.submodule ({ config, ... }: { + options.krebs.build.source = let + raw = types.either types.str types.path; + url = types.submodule { options = { - host = mkOption { - type = types.host; - default = default-host; - }; - path = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; url = mkOption { type = types.str; - default = "file://${config.host.name}${config.path}"; - }; - }; - })); - default = {}; - }; - - options.krebs.build.source.git = mkOption { - type = with types; attrsOf (submodule ({ config, ... }: { - options = { - url = mkOption { - type = types.str; # TODO must be shell safe }; rev = mkOption { type = types.str; }; - target-path = mkOption { + dev = mkOption { type = types.str; - default = "/root/${config._module.args.name}"; }; }; - })); + }; + in mkOption { + type = types.attrsOf (types.either types.str url); + apply = let f = mapAttrs (_: value: { + string = value; + path = toString value; + set = f value; + }.${typeOf value}); in f; default = {}; }; + + options.krebs.build.populate = mkOption { + type = types.str; + default = let + source = config.krebs.build.source; + target-user = maybeEnv "target_user" "root"; + target-host = maybeEnv "target_host" config.krebs.build.host.name; + target-path = maybeEnv "target_path" "/var/src"; + out = '' + #! /bin/sh + set -eu + + verbose() { + printf '+%s\n' "$(printf ' %q' "$@")" >&2 + "$@" + } + + echo ${shell.escape git-script} \ + | ssh ${shell.escape "${target-user}@${target-host}"} -T + + unset tmpdir + trap ' + rm "$tmpdir"/* + rmdir "$tmpdir" + trap - EXIT INT QUIT + ' EXIT INT QUIT + tmpdir=$(mktemp -dt stockholm.XXXXXXXX) + chmod 0755 "$tmpdir" + + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: let dst = removePrefix "symlink:" (get-url spec); in + "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") + symlink-specs)} + + verbose proot \ + -b $tmpdir:${shell.escape target-path} \ + ${concatStringsSep " \\\n " + (mapAttrsToList + (name: spec: + "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}") + file-specs)} \ + rsync \ + -f ${shell.escape "P /*"} \ + ${concatMapStringsSep " \\\n " + (name: "-f ${shell.escape "R /${name}"}") + (attrNames file-specs)} \ + --delete \ + -vFrlptD \ + ${shell.escape target-path}/ \ + ${shell.escape "${target-user}@${target-host}:${target-path}"} + ''; + + get-schema = uri: + if substring 0 1 uri == "/" + then "file" + else head (splitString ":" uri); + + has-schema = schema: uri: get-schema uri == schema; + + get-url = spec: { + string = spec; + path = toString spec; + set = get-url spec.url; + }.${typeOf spec}; + + git-specs = + filterAttrs (_: spec: has-schema "https" (get-url spec)) source // + filterAttrs (_: spec: has-schema "http" (get-url spec)) source // + filterAttrs (_: spec: has-schema "git" (get-url spec)) source; + + file-specs = + filterAttrs (_: spec: has-schema "file" (get-url spec)) source; + + symlink-specs = + filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source; + + git-script = '' + #! /bin/sh + set -efu + + verbose() { + printf '+%s\n' "$(printf ' %q' "$@")" >&2 + "$@" + } + + fetch_git() {( + dst_dir=$1 + src_url=$2 + src_ref=$3 + + if ! test -e "$dst_dir"; then + git clone "$src_url" "$dst_dir" + fi + + cd "$dst_dir" + + if ! url=$(git config remote.origin.url); then + git remote add origin "$src_url" + elif test "$url" != "$src_url"; then + git remote set-url origin "$src_url" + fi + + # TODO resolve src_ref to commit hash + hash=$src_ref + + if ! test "$(git log --format=%H -1)" = "$hash"; then + git fetch origin + git checkout "$hash" -- "$dst_dir" + git checkout "$hash" + fi + + git clean -dxf + )} + + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: toString (map shell.escape [ + "verbose" + "fetch_git" + "${target-path}/${name}" + spec.url + spec.rev + ])) + git-specs)} + ''; + in out; + }; + }; in out diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix index e6267d7e6..7b28ffca8 100644 --- a/krebs/3modules/git.nix +++ b/krebs/3modules/git.nix @@ -27,7 +27,7 @@ let description = '' Enable cgit. Cgit is an attempt to create a fast web interface for the git version - control system, using a built in cache to decrease pressure on the + control system, using a built in cache to decrease pressure on the git server. cgit in this module is being served via fastcgi nginx.This module deploys a http://cgit.<hostname> nginx configuration and enables nginx @@ -44,48 +44,8 @@ let default = "/etc/git"; }; repos = mkOption { - type = types.attrsOf (types.submodule ({ - options = { - desc = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - Repository description. - ''; - }; - section = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - Repository section. - ''; - }; - name = mkOption { - type = types.str; - description = '' - Repository name. - ''; - }; - hooks = mkOption { - type = types.attrsOf types.str; - default = {}; - description = '' - Repository-specific hooks. - ''; - }; - public = mkOption { - type = types.bool; - default = false; - description = '' - Allow everybody to read the repository via HTTP if cgit enabled. - ''; - # TODO allow every configured user to fetch the repository via SSH. - }; - }; - })); - + type = types.attrsOf subtypes.repo; default = {}; - example = literalExample '' { testing = { @@ -99,7 +59,6 @@ let testing2 = { name = "testing2"; }; } ''; - description = '' Repositories. ''; @@ -121,30 +80,158 @@ let ''; }; rules = mkOption { - type = types.unspecified; + type = types.listOf subtypes.rule; + default = []; + example = literalExample '' + singleton { + user = [ config.krebs.users.tv ]; + repo = [ testing ]; # see literal example of repos + perm = push "refs/*" (with lib.git; [ + non-fast-forward create delete merge + ]); + } + ''; + description = '' + Rules. + ''; }; }; + # TODO put into krebs/4lib/types.nix? + subtypes = { + repo = types.submodule ({ + options = { + collaborators = mkOption { + type = types.listOf types.user; + default = []; + description = '' + List of users that should be able to fetch from this repo. + + This option is currently not used by krebs.git but instead can be + used to create rules. See e.g. <stockholm/tv/2configs/git.nix> for + an example. + ''; + }; + desc = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Repository description. + ''; + }; + section = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Repository section. + ''; + }; + name = mkOption { + type = types.str; + description = '' + Repository name. + ''; + }; + hooks = mkOption { + type = types.attrsOf types.str; + default = {}; + description = '' + Repository-specific hooks. + ''; + }; + public = mkOption { + type = types.bool; + default = false; + description = '' + Allow everybody to read the repository via HTTP if cgit enabled. + ''; + # TODO allow every configured user to fetch the repository via SSH. + }; + }; + }); + rule = types.submodule ({ config, ... }: { + options = { + user = mkOption { + type = types.listOf types.user; + description = '' + List of users this rule should apply to. + Checked by authorize-command. + ''; + }; + repo = mkOption { + type = types.listOf subtypes.repo; + description = '' + List of repos this rule should apply to. + Checked by authorize-command. + ''; + }; + perm = mkOption { + type = types.submodule { + # TODO generate enum argument from krebs/4lib/git.nix + options = { + allow-commands = mkOption { + type = types.listOf (types.enum (with git; [ + git-receive-pack + git-upload-pack + ])); + default = []; + description = '' + List of commands the rule's users are allowed to execute. + Checked by authorize-command. + ''; + }; + allow-receive-ref = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Ref that can receive objects. + Checked by authorize-push. + ''; + }; + allow-receive-modes = mkOption { + type = types.listOf (types.enum (with git; [ + fast-forward + non-fast-forward + create + delete + merge + ])); + default = []; + description = '' + List of allowed receive modes. + Checked by pre-receive hook. + ''; + }; + }; + }; + description = '' + Permissions granted. + ''; + }; + }; + }); + }; + git-imp = { system.activationScripts.git-init = "${init-script}"; - + # TODO maybe put all scripts here and then use PATH? environment.etc."${etc-base}".source = scriptFarm "git-ssh-authorizers" { - authorize-command = makeAuthorizeScript (map ({ repo, user, perm }: [ - (map getName (ensureList user)) - (map getName (ensureList repo)) - (map getName perm.allow-commands) + authorize-command = makeAuthorizeScript (map (rule: [ + (map getName (ensureList rule.user)) + (map getName (ensureList rule.repo)) + (map getName rule.perm.allow-commands) ]) cfg.rules); - - authorize-push = makeAuthorizeScript (map ({ repo, user, perm }: [ - (map getName (ensureList user)) - (map getName (ensureList repo)) - (ensureList perm.allow-receive-ref) - (map getName perm.allow-receive-modes) - ]) (filter (x: hasAttr "allow-receive-ref" x.perm) cfg.rules)); + + authorize-push = makeAuthorizeScript (map (rule: [ + (map getName (ensureList rule.user)) + (map getName (ensureList rule.repo)) + (ensureList rule.perm.allow-receive-ref) + (map getName rule.perm.allow-receive-modes) + ]) (filter (rule: rule.perm.allow-receive-ref != null) cfg.rules)); }; |