diff options
Diffstat (limited to 'krebs/3modules')
| -rw-r--r-- | krebs/3modules/backup.nix | 22 | ||||
| -rw-r--r-- | krebs/3modules/build.nix | 171 | ||||
| -rw-r--r-- | krebs/3modules/git.nix | 201 | ||||
| -rw-r--r-- | krebs/3modules/tv/default.nix | 1 | ||||
| -rw-r--r-- | krebs/3modules/urlwatch.nix | 46 |
5 files changed, 351 insertions, 90 deletions
diff --git a/krebs/3modules/backup.nix b/krebs/3modules/backup.nix index 01bb16a2b..a1f335905 100644 --- a/krebs/3modules/backup.nix +++ b/krebs/3modules/backup.nix @@ -28,9 +28,17 @@ let type = types.krebs.file-location; }; startAt = mkOption { - type = types.str; + default = "hourly"; + type = types.str; # TODO systemd.time(7)'s calendar event }; snapshots = mkOption { + default = { + hourly = { format = "%Y-%m-%dT%H"; retain = 4; }; + daily = { format = "%Y-%m-%d"; retain = 7; }; + weekly = { format = "%YW%W"; retain = 4; }; + monthly = { format = "%Y-%m"; retain = 12; }; + yearly = { format = "%Y"; }; + }; type = types.attrsOf (types.submodule { options = { format = mkOption { @@ -284,3 +292,15 @@ let }; in out +# TODO ionice +# TODO mail on failed push, pull +# TODO mail on missing push +# TODO don't cancel plans on activation +# also, don't hang while deploying at: +# starting the following units: backup.wu-home-xu.push.service, backup.wu-home-xu.push.timer +# TODO make sure /bku is properly mounted +# TODO make sure that secure hosts cannot backup to insecure ones +# TODO optionally only backup when src and dst are near enough :) +# TODO try using btrfs for snapshots (configurable) +# TODO warn if partial snapshots are found +# TODO warn if unknown stuff is found in dst path diff --git a/krebs/3modules/build.nix b/krebs/3modules/build.nix index 7f004cd81..00142acdd 100644 --- a/krebs/3modules/build.nix +++ b/krebs/3modules/build.nix @@ -28,48 +28,159 @@ let type = types.user; }; - options.krebs.build.source.dir = mkOption { - type = let - default-host = config.krebs.current.host; - in types.attrsOf (types.submodule ({ config, ... }: { + options.krebs.build.source = let + raw = types.either types.str types.path; + url = types.submodule { options = { - host = mkOption { - type = types.host; - default = default-host; - }; - path = mkOption { - type = types.str; - }; - target-path = mkOption { - type = types.str; - default = "/root/${config._module.args.name}"; - }; url = mkOption { type = types.str; - default = "file://${config.host.name}${config.path}"; - }; - }; - })); - default = {}; - }; - - options.krebs.build.source.git = mkOption { - type = with types; attrsOf (submodule ({ config, ... }: { - options = { - url = mkOption { - type = types.str; # TODO must be shell safe }; rev = mkOption { type = types.str; }; - target-path = mkOption { + dev = mkOption { type = types.str; - default = "/root/${config._module.args.name}"; }; }; - })); + }; + in mkOption { + type = types.attrsOf (types.either types.str url); + apply = let f = mapAttrs (_: value: { + string = value; + path = toString value; + set = f value; + }.${typeOf value}); in f; default = {}; }; + + options.krebs.build.populate = mkOption { + type = types.str; + default = let + source = config.krebs.build.source; + target-user = maybeEnv "target_user" "root"; + target-host = maybeEnv "target_host" config.krebs.build.host.name; + target-path = maybeEnv "target_path" "/var/src"; + out = '' + #! /bin/sh + set -eu + + verbose() { + printf '+%s\n' "$(printf ' %q' "$@")" >&2 + "$@" + } + + echo ${shell.escape git-script} \ + | ssh ${shell.escape "${target-user}@${target-host}"} -T + + unset tmpdir + trap ' + rm "$tmpdir"/* + rmdir "$tmpdir" + trap - EXIT INT QUIT + ' EXIT INT QUIT + tmpdir=$(mktemp -dt stockholm.XXXXXXXX) + chmod 0755 "$tmpdir" + + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: let dst = removePrefix "symlink:" (get-url spec); in + "verbose ln -s ${shell.escape dst} $tmpdir/${shell.escape name}") + symlink-specs)} + + verbose proot \ + -b $tmpdir:${shell.escape target-path} \ + ${concatStringsSep " \\\n " + (mapAttrsToList + (name: spec: + "-b ${shell.escape "${get-url spec}:${target-path}/${name}"}") + file-specs)} \ + rsync \ + -f ${shell.escape "P /*"} \ + ${concatMapStringsSep " \\\n " + (name: "-f ${shell.escape "R /${name}"}") + (attrNames file-specs)} \ + --delete \ + -vFrlptD \ + ${shell.escape target-path}/ \ + ${shell.escape "${target-user}@${target-host}:${target-path}"} + ''; + + get-schema = uri: + if substring 0 1 uri == "/" + then "file" + else head (splitString ":" uri); + + has-schema = schema: uri: get-schema uri == schema; + + get-url = spec: { + string = spec; + path = toString spec; + set = get-url spec.url; + }.${typeOf spec}; + + git-specs = + filterAttrs (_: spec: has-schema "https" (get-url spec)) source // + filterAttrs (_: spec: has-schema "http" (get-url spec)) source // + filterAttrs (_: spec: has-schema "git" (get-url spec)) source; + + file-specs = + filterAttrs (_: spec: has-schema "file" (get-url spec)) source; + + symlink-specs = + filterAttrs (_: spec: has-schema "symlink" (get-url spec)) source; + + git-script = '' + #! /bin/sh + set -efu + + verbose() { + printf '+%s\n' "$(printf ' %q' "$@")" >&2 + "$@" + } + + fetch_git() {( + dst_dir=$1 + src_url=$2 + src_ref=$3 + + if ! test -e "$dst_dir"; then + git clone "$src_url" "$dst_dir" + fi + + cd "$dst_dir" + + if ! url=$(git config remote.origin.url); then + git remote add origin "$src_url" + elif test "$url" != "$src_url"; then + git remote set-url origin "$src_url" + fi + + # TODO resolve src_ref to commit hash + hash=$src_ref + + if ! test "$(git log --format=%H -1)" = "$hash"; then + git fetch origin + git checkout "$hash" -- "$dst_dir" + git checkout "$hash" + fi + + git clean -dxf + )} + + ${concatStringsSep "\n" + (mapAttrsToList + (name: spec: toString (map shell.escape [ + "verbose" + "fetch_git" + "${target-path}/${name}" + spec.url + spec.rev + ])) + git-specs)} + ''; + in out; + }; + }; in out diff --git a/krebs/3modules/git.nix b/krebs/3modules/git.nix index e6267d7e6..7b28ffca8 100644 --- a/krebs/3modules/git.nix +++ b/krebs/3modules/git.nix @@ -27,7 +27,7 @@ let description = '' Enable cgit. Cgit is an attempt to create a fast web interface for the git version - control system, using a built in cache to decrease pressure on the + control system, using a built in cache to decrease pressure on the git server. cgit in this module is being served via fastcgi nginx.This module deploys a http://cgit.<hostname> nginx configuration and enables nginx @@ -44,48 +44,8 @@ let default = "/etc/git"; }; repos = mkOption { - type = types.attrsOf (types.submodule ({ - options = { - desc = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - Repository description. - ''; - }; - section = mkOption { - type = types.nullOr types.str; - default = null; - description = '' - Repository section. - ''; - }; - name = mkOption { - type = types.str; - description = '' - Repository name. - ''; - }; - hooks = mkOption { - type = types.attrsOf types.str; - default = {}; - description = '' - Repository-specific hooks. - ''; - }; - public = mkOption { - type = types.bool; - default = false; - description = '' - Allow everybody to read the repository via HTTP if cgit enabled. - ''; - # TODO allow every configured user to fetch the repository via SSH. - }; - }; - })); - + type = types.attrsOf subtypes.repo; default = {}; - example = literalExample '' { testing = { @@ -99,7 +59,6 @@ let testing2 = { name = "testing2"; }; } ''; - description = '' Repositories. ''; @@ -121,30 +80,158 @@ let ''; }; rules = mkOption { - type = types.unspecified; + type = types.listOf subtypes.rule; + default = []; + example = literalExample '' + singleton { + user = [ config.krebs.users.tv ]; + repo = [ testing ]; # see literal example of repos + perm = push "refs/*" (with lib.git; [ + non-fast-forward create delete merge + ]); + } + ''; + description = '' + Rules. + ''; }; }; + # TODO put into krebs/4lib/types.nix? + subtypes = { + repo = types.submodule ({ + options = { + collaborators = mkOption { + type = types.listOf types.user; + default = []; + description = '' + List of users that should be able to fetch from this repo. + + This option is currently not used by krebs.git but instead can be + used to create rules. See e.g. <stockholm/tv/2configs/git.nix> for + an example. + ''; + }; + desc = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Repository description. + ''; + }; + section = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Repository section. + ''; + }; + name = mkOption { + type = types.str; + description = '' + Repository name. + ''; + }; + hooks = mkOption { + type = types.attrsOf types.str; + default = {}; + description = '' + Repository-specific hooks. + ''; + }; + public = mkOption { + type = types.bool; + default = false; + description = '' + Allow everybody to read the repository via HTTP if cgit enabled. + ''; + # TODO allow every configured user to fetch the repository via SSH. + }; + }; + }); + rule = types.submodule ({ config, ... }: { + options = { + user = mkOption { + type = types.listOf types.user; + description = '' + List of users this rule should apply to. + Checked by authorize-command. + ''; + }; + repo = mkOption { + type = types.listOf subtypes.repo; + description = '' + List of repos this rule should apply to. + Checked by authorize-command. + ''; + }; + perm = mkOption { + type = types.submodule { + # TODO generate enum argument from krebs/4lib/git.nix + options = { + allow-commands = mkOption { + type = types.listOf (types.enum (with git; [ + git-receive-pack + git-upload-pack + ])); + default = []; + description = '' + List of commands the rule's users are allowed to execute. + Checked by authorize-command. + ''; + }; + allow-receive-ref = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Ref that can receive objects. + Checked by authorize-push. + ''; + }; + allow-receive-modes = mkOption { + type = types.listOf (types.enum (with git; [ + fast-forward + non-fast-forward + create + delete + merge + ])); + default = []; + description = '' + List of allowed receive modes. + Checked by pre-receive hook. + ''; + }; + }; + }; + description = '' + Permissions granted. + ''; + }; + }; + }); + }; + git-imp = { system.activationScripts.git-init = "${init-script}"; - + # TODO maybe put all scripts here and then use PATH? environment.etc."${etc-base}".source = scriptFarm "git-ssh-authorizers" { - authorize-command = makeAuthorizeScript (map ({ repo, user, perm }: [ - (map getName (ensureList user)) - (map getName (ensureList repo)) - (map getName perm.allow-commands) + authorize-command = makeAuthorizeScript (map (rule: [ + (map getName (ensureList rule.user)) + (map getName (ensureList rule.repo)) + (map getName rule.perm.allow-commands) ]) cfg.rules); - - authorize-push = makeAuthorizeScript (map ({ repo, user, perm }: [ - (map getName (ensureList user)) - (map getName (ensureList repo)) - (ensureList perm.allow-receive-ref) - (map getName perm.allow-receive-modes) - ]) (filter (x: hasAttr "allow-receive-ref" x.perm) cfg.rules)); + + authorize-push = makeAuthorizeScript (map (rule: [ + (map getName (ensureList rule.user)) + (map getName (ensureList rule.repo)) + (ensureList rule.perm.allow-receive-ref) + (map getName rule.perm.allow-receive-modes) + ]) (filter (rule: rule.perm.allow-receive-ref != null) cfg.rules)); }; - + users.extraUsers = singleton rec { description = "Git repository hosting user"; name = "git"; diff --git a/krebs/3modules/tv/default.nix b/krebs/3modules/tv/default.nix index 6fd1c4224..31c1a375a 100644 --- a/krebs/3modules/tv/default.nix +++ b/krebs/3modules/tv/default.nix @@ -247,6 +247,7 @@ with lib; }; }; secure = true; + ssh.privkey.path = <secrets/ssh.id_ed25519>; ssh.pubkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIcJvu8JDVzObLUtlAQg9qVugthKSfitwCljuJ5liyHa"; }; xu = { diff --git a/krebs/3modules/urlwatch.nix b/krebs/3modules/urlwatch.nix index 31cbfcf6e..7a9fb55fd 100644 --- a/krebs/3modules/urlwatch.nix +++ b/krebs/3modules/urlwatch.nix @@ -54,6 +54,10 @@ let example = [ https://nixos.org/channels/nixos-unstable/git-revision ]; + apply = map (x: getAttr (typeOf x) { + set = x; + string.url = x; + }); }; verbose = mkOption { type = types.bool; @@ -64,7 +68,40 @@ let }; }; - urlsFile = toFile "urls" (concatStringsSep "\n" cfg.urls); + urlsFile = toFile "urls" (concatMapStringsSep "\n---\n" toJSON cfg.urls); + + configFile = toFile "urlwatch.yaml" (toJSON { + display = { + error = true; + new = true; + unchanged = false; + }; + report = { + email = { + enabled = false; + from = ""; + html = false; + smtp = { + host = "localhost"; + keyring = true; + port = 25; + starttls = true; + }; + subject = "{count} changes: {jobs}"; + to = ""; + }; + html.diff = "unified"; + stdout = { + color = true; + enabled = true; + }; + text = { + details = true; + footer = true; + line_length = 75; + }; + }; + }); imp = { systemd.timers.urlwatch = { @@ -109,10 +146,15 @@ let from=${escapeShellArg cfg.from} mailto=${escapeShellArg cfg.mailto} urlsFile=${escapeShellArg urlsFile} + configFile=${escapeShellArg configFile} cd /tmp - urlwatch -e ${optionalString cfg.verbose "-v"} --urls="$urlsFile" > changes || : + urlwatch \ + ${optionalString cfg.verbose "-v"} \ + --urls="$urlsFile" \ + --config="$configFile" \ + > changes || : if test -s changes; then date=$(date -R) |